{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import math\n",
    "import multiprocessing as mp\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torchvision import datasets, transforms\n",
    "from torch.autograd import Variable\n",
    "from collections import namedtuple\n",
    "\n",
    "from PIL import Image\n",
    "import os\n",
    "import os.path\n",
    "import errno\n",
    "import codecs\n",
    "import copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.cuda.device_count() 4\n",
      "torch.cuda.current_device() 0\n",
      "torch.cuda.current_device() 3\n"
     ]
    }
   ],
   "source": [
    "torch.manual_seed(0)\n",
    "np.random.seed(0)\n",
    "print(\"torch.cuda.device_count()\", torch.cuda.device_count())\n",
    "print(\"torch.cuda.current_device()\", torch.cuda.current_device())\n",
    "torch.cuda.set_device(3)\n",
    "print(\"torch.cuda.current_device()\", torch.cuda.current_device())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def compute_ranks(x):\n",
    "  \"\"\"\n",
    "  Returns ranks in [0, len(x))\n",
    "  Note: This is different from scipy.stats.rankdata, which returns ranks in [1, len(x)].\n",
    "  (https://github.com/openai/evolution-strategies-starter/blob/master/es_distributed/es.py)\n",
    "  \"\"\"\n",
    "  assert x.ndim == 1\n",
    "  ranks = np.empty(len(x), dtype=int)\n",
    "  ranks[x.argsort()] = np.arange(len(x))\n",
    "  return ranks\n",
    "\n",
    "def compute_centered_ranks(x):\n",
    "  \"\"\"\n",
    "  https://github.com/openai/evolution-strategies-starter/blob/master/es_distributed/es.py\n",
    "  \"\"\"\n",
    "  y = compute_ranks(x.ravel()).reshape(x.shape).astype(np.float32)\n",
    "  y /= (x.size - 1)\n",
    "  y -= .5\n",
    "  return y\n",
    "\n",
    "def compute_weight_decay(weight_decay, model_param_list):\n",
    "  model_param_grid = np.array(model_param_list)\n",
    "  return - weight_decay * np.mean(model_param_grid * model_param_grid, axis=1)\n",
    "\n",
    "class CMAES:\n",
    "  '''CMA-ES wrapper.'''\n",
    "  def __init__(self, num_params,      # number of model parameters\n",
    "               sigma_init=0.10,       # initial standard deviation\n",
    "               popsize=255):          # population size\n",
    "\n",
    "    self.num_params = num_params\n",
    "    self.sigma_init = sigma_init\n",
    "    self.popsize = popsize\n",
    "\n",
    "    self.solutions = None\n",
    "\n",
    "    import cma\n",
    "    self.es = cma.CMAEvolutionStrategy( self.num_params * [0],\n",
    "                                        self.sigma_init,\n",
    "                                        {'popsize': self.popsize})\n",
    "\n",
    "  def rms_stdev(self):\n",
    "    sigma = self.es.result[6]\n",
    "    return np.mean(np.sqrt(sigma*sigma))\n",
    "\n",
    "  def ask(self):\n",
    "    '''returns a list of parameters'''\n",
    "    self.solutions = np.array(self.es.ask())\n",
    "    return self.solutions\n",
    "\n",
    "  def tell(self, reward_table_result):\n",
    "    reward_table = reward_table_result\n",
    "    self.es.tell(self.solutions, (-reward_table).tolist()) # convert minimizer to maximizer.\n",
    "\n",
    "  def done(self):\n",
    "    return self.es.stop()\n",
    "\n",
    "  def current_param(self):\n",
    "    return self.es.result[5] # mean solution, presumably better with noise\n",
    "  \n",
    "  def best_param(self):\n",
    "    return self.es.result[0] # best evaluated solution\n",
    "\n",
    "  def result(self): # return best params so far, along with historically best reward, curr reward, sigma\n",
    "    r = self.es.result\n",
    "    return (r[0], -r[1], -r[1], r[6])\n",
    "\n",
    "class SimpleES:\n",
    "  '''Simple Evolution Strategies.'''\n",
    "  def __init__(self, num_params,      # number of model parameters\n",
    "               sigma_init=0.10,       # initial standard deviation\n",
    "               sigma_alpha=0.20,      # learning rate for standard deviation\n",
    "               sigma_decay=0.999,     # anneal standard deviation\n",
    "               sigma_limit=0.01,      # stop annealing if less than this\n",
    "               popsize=255,           # population size\n",
    "               elite_ratio=0.1,       # percentage of the elites\n",
    "               done_threshold=1e-6,   # threshold when we say we are done\n",
    "               average_baseline=True, # set baseline to average of batch\n",
    "               forget_best=True):     # only use the best from latest generation\n",
    "\n",
    "    self.num_params = num_params\n",
    "    self.sigma_init = sigma_init\n",
    "    self.sigma_alpha = sigma_alpha\n",
    "    self.sigma_decay = sigma_decay\n",
    "    self.sigma_limit = sigma_limit\n",
    "    self.popsize = popsize\n",
    "    self.average_baseline = average_baseline\n",
    "    if self.average_baseline:\n",
    "      assert (self.popsize & 2), \"Population size must be even\"\n",
    "      self.batch_size = int(self.popsize / 2)\n",
    "    else:\n",
    "      assert (self.popsize & 1), \"Population size must be odd\"\n",
    "      self.batch_size = int((self.popsize - 1) / 2)\n",
    "    self.elite_ratio = elite_ratio\n",
    "    self.elite_popsize = int(self.popsize * self.elite_ratio)\n",
    "    self.forget_best = forget_best\n",
    "    self.batch_reward = np.zeros(self.batch_size * 2)\n",
    "    self.mu = np.zeros(self.num_params)\n",
    "    self.sigma = np.ones(self.num_params) * self.sigma_init\n",
    "    self.curr_best_mu = np.zeros(self.num_params)\n",
    "    self.best_mu = np.zeros(self.num_params)\n",
    "    self.best_reward = 0\n",
    "    self.first_interation = True\n",
    "    self.done_threshold = done_threshold\n",
    "\n",
    "  def rms_stdev(self):\n",
    "    sigma = self.sigma\n",
    "    return np.mean(np.sqrt(sigma*sigma))\n",
    "\n",
    "  def ask(self):\n",
    "    '''returns a list of parameters'''\n",
    "    # antithetic sampling\n",
    "    self.epsilon = np.random.randn(self.batch_size, self.num_params) * self.sigma.reshape(1, self.num_params)\n",
    "    self.epsilon_full = np.concatenate([self.epsilon, - self.epsilon])\n",
    "    if self.average_baseline:\n",
    "      epsilon = self.epsilon_full\n",
    "    else:\n",
    "      # first population is mu, then positive epsilon, then negative epsilon\n",
    "      epsilon = np.concatenate([np.zeros((1, self.num_params)), self.epsilon_full])\n",
    "    solutions = self.mu.reshape(1, self.num_params) + epsilon\n",
    "    return solutions\n",
    "\n",
    "  def tell(self, reward_table_result):\n",
    "    # input must be a numpy float array\n",
    "    assert(len(reward_table_result) == self.popsize), \"Inconsistent reward_table size reported.\"\n",
    "\n",
    "    reward_table = reward_table_result\n",
    "\n",
    "    reward_offset = 1\n",
    "    if self.average_baseline:\n",
    "      b = np.mean(reward_table)\n",
    "      reward_offset = 0\n",
    "    else:\n",
    "      b = reward_table[0] # baseline\n",
    "      \n",
    "    reward = reward_table[reward_offset:]\n",
    "    idx = np.argsort(reward)[::-1][0:self.elite_popsize]\n",
    "\n",
    "    best_reward = reward[idx[0]]\n",
    "    if (best_reward > b or self.average_baseline):\n",
    "      best_mu = self.mu + self.epsilon_full[idx[0]]\n",
    "      best_reward = reward[idx[0]]\n",
    "    else:\n",
    "      best_mu = self.mu\n",
    "      best_reward = b\n",
    "\n",
    "    self.curr_best_reward = best_reward\n",
    "    self.curr_best_mu = best_mu\n",
    "\n",
    "    if self.first_interation:\n",
    "      self.first_interation = False\n",
    "      self.best_reward = self.curr_best_reward\n",
    "      self.best_mu = best_mu\n",
    "    else:\n",
    "      if self.forget_best or (self.curr_best_reward > self.best_reward):\n",
    "        self.best_mu = best_mu\n",
    "        self.best_reward = self.curr_best_reward\n",
    "\n",
    "    # adaptive sigma\n",
    "    # normalization\n",
    "    stdev_reward = reward.std()\n",
    "    epsilon = self.epsilon\n",
    "    sigma = self.sigma\n",
    "    S = ((epsilon * epsilon - (sigma * sigma).reshape(1, self.num_params)) / sigma.reshape(1, self.num_params))\n",
    "    reward_avg = (reward[:self.batch_size] + reward[self.batch_size:]) / 2.0\n",
    "    rS = reward_avg - b\n",
    "    delta_sigma = (np.dot(rS, S)) / (2 * self.batch_size * stdev_reward)\n",
    "\n",
    "    # move mean to the average of the best idx means\n",
    "    self.mu += self.epsilon_full[idx].mean(axis=0)\n",
    "\n",
    "    # adjust sigma according to the adaptive sigma calculation\n",
    "    change_sigma = self.sigma_alpha * delta_sigma\n",
    "    change_sigma = np.minimum(change_sigma, self.sigma)\n",
    "    change_sigma = np.maximum(change_sigma, - 0.5 * self.sigma)\n",
    "    self.sigma += change_sigma\n",
    "    self.sigma[self.sigma > self.sigma_limit] *= self.sigma_decay\n",
    "\n",
    "  def done(self):\n",
    "    return (self.rms_stdev() < self.done_threshold)\n",
    "\n",
    "  def current_param(self):\n",
    "    return self.curr_best_mu\n",
    "  \n",
    "  def best_param(self):\n",
    "    return self.best_mu\n",
    "\n",
    "  def result(self): # return best params so far, along with historically best reward, curr reward, sigma\n",
    "    return (self.best_mu, self.best_reward, self.curr_best_reward, self.sigma)\n",
    "\n",
    "class SimpleGA:\n",
    "  '''Simple Genetic Algorithm.'''\n",
    "  def __init__(self, num_params,      # number of model parameters\n",
    "               sigma_init=0.1,        # initial standard deviation\n",
    "               sigma_decay=0.999,     # anneal standard deviation\n",
    "               sigma_limit=0.01,      # stop annealing if less than this\n",
    "               popsize=255,           # population size\n",
    "               elite_ratio=0.1,       # percentage of the elites\n",
    "               forget_best=False,     # forget the historical best elites\n",
    "               done_threshold=1e-6):  # threshold when we say we are done\n",
    "\n",
    "    self.num_params = num_params\n",
    "    self.sigma_init = sigma_init\n",
    "    self.sigma_decay = sigma_decay\n",
    "    self.sigma_limit = sigma_limit\n",
    "    self.popsize = popsize\n",
    "\n",
    "    self.elite_ratio = elite_ratio\n",
    "    self.elite_popsize = int(self.popsize * self.elite_ratio)\n",
    "\n",
    "    self.sigma = self.sigma_init\n",
    "    self.elite_params = np.zeros((self.elite_popsize, self.num_params))\n",
    "    self.elite_rewards = np.zeros(self.elite_popsize)\n",
    "    self.best_param = np.zeros(self.num_params)\n",
    "    self.best_reward = 0\n",
    "    self.first_iteration = True\n",
    "    self.forget_best = forget_best\n",
    "    self.done_threshold = done_threshold\n",
    "\n",
    "  def rms_stdev(self):\n",
    "    return self.sigma # same sigma for all parameters.\n",
    "\n",
    "  def ask(self):\n",
    "    '''returns a list of parameters'''\n",
    "    # antithetic sampling\n",
    "    self.epsilon = np.random.randn(self.popsize, self.num_params) * self.sigma\n",
    "    solutions = []\n",
    "    \n",
    "    def mate(a, b):\n",
    "      c = np.copy(a)\n",
    "      idx = np.where(np.random.rand((c.size)) > 0.5)\n",
    "      c[idx] = b[idx]\n",
    "      return c\n",
    "    \n",
    "    elite_range = range(self.elite_popsize)\n",
    "    for i in range(self.popsize):\n",
    "      idx_a = np.random.choice(elite_range)\n",
    "      idx_b = np.random.choice(elite_range)\n",
    "      child_params = mate(self.elite_params[idx_a], self.elite_params[idx_b])\n",
    "      solutions.append(child_params + self.epsilon[i])\n",
    "\n",
    "    solutions = np.array(solutions)\n",
    "    self.solutions = solutions\n",
    "\n",
    "    return solutions\n",
    "\n",
    "  def tell(self, reward_table_result):\n",
    "    # input must be a numpy float array\n",
    "    assert(len(reward_table_result) == self.popsize), \"Inconsistent reward_table size reported.\"\n",
    "    \n",
    "    if (not self.forget_best or self.first_iteration):\n",
    "      reward = reward_table_result\n",
    "      solution = self.solutions\n",
    "    else:\n",
    "      reward = np.concatenate([reward_table_result, self.elite_rewards])\n",
    "      solution = np.concatenate([self.solutions, self.elite_params])\n",
    "\n",
    "    idx = np.argsort(reward)[::-1][0:self.elite_popsize]\n",
    "\n",
    "    self.elite_rewards = reward[idx]\n",
    "    self.elite_params = solution[idx]\n",
    "\n",
    "    self.curr_best_reward = self.elite_rewards[0]\n",
    "    \n",
    "    if self.first_iteration or (self.curr_best_reward > self.best_reward):\n",
    "      self.first_iteration = False\n",
    "      self.best_reward = self.elite_rewards[0]\n",
    "      self.best_param = np.copy(self.elite_params[0])\n",
    "\n",
    "    if (self.sigma > self.sigma_limit):\n",
    "      self.sigma *= self.sigma_decay\n",
    "\n",
    "  def done(self):\n",
    "    return (self.rms_stdev() < self.done_threshold)\n",
    "\n",
    "  def current_param(self):\n",
    "    return self.elite_params[0]\n",
    "\n",
    "  def best_param(self):\n",
    "    return self.best_param\n",
    "\n",
    "  def result(self): # return best params so far, along with historically best reward, curr reward, sigma\n",
    "    return (self.best_param, self.best_reward, self.curr_best_reward, self.sigma)\n",
    "\n",
    "class OpenES:\n",
    "  ''' Basic Version of OpenAI Evolution Strategies.'''\n",
    "  def __init__(self, num_params,             # number of model parameters\n",
    "               sigma_init=0.1,               # initial standard deviation\n",
    "               sigma_decay=0.999,            # anneal standard deviation\n",
    "               sigma_limit=0.01,             # stop annealing if less than this\n",
    "               learning_rate=0.001,          # learning rate for standard deviation\n",
    "               learning_rate_decay = 0.9999, # annealing the learning rate\n",
    "               learning_rate_limit = 0.001,  # stop annealing learning rate\n",
    "               popsize=255,                  # population size\n",
    "               antithetic=False,             # whether to use antithetic sampling\n",
    "               forget_best=True):           # forget historical best\n",
    "\n",
    "    self.num_params = num_params\n",
    "    self.sigma_decay = sigma_decay\n",
    "    self.sigma = sigma_init\n",
    "    self.sigma_limit = sigma_limit\n",
    "    self.learning_rate = learning_rate\n",
    "    self.learning_rate_decay = learning_rate_decay\n",
    "    self.learning_rate_limit = learning_rate_limit\n",
    "    self.popsize = popsize\n",
    "    self.antithetic = antithetic\n",
    "    if self.antithetic:\n",
    "      assert (self.popsize & 2), \"Population size must be even\"\n",
    "      self.half_popsize = int(self.popsize / 2)\n",
    "\n",
    "    self.reward = np.zeros(self.popsize)\n",
    "    self.mu = np.zeros(self.num_params)\n",
    "    self.best_mu = np.zeros(self.num_params)\n",
    "    self.best_reward = 0\n",
    "    self.first_interation = True\n",
    "    self.forget_best = forget_best\n",
    "\n",
    "  def rms_stdev(self):\n",
    "    sigma = self.sigma\n",
    "    return np.mean(np.sqrt(sigma*sigma))\n",
    "\n",
    "  def ask(self):\n",
    "    '''returns a list of parameters'''\n",
    "    # antithetic sampling\n",
    "    if self.antithetic:\n",
    "      self.epsilon_half = np.random.randn(self.half_popsize, self.num_params)\n",
    "      self.epsilon = np.concatenate([self.epsilon_half, - self.epsilon_half])\n",
    "    else:\n",
    "      self.epsilon = np.random.randn(self.popsize, self.num_params)\n",
    "\n",
    "    self.solutions = self.mu.reshape(1, self.num_params) + self.epsilon * self.sigma\n",
    "\n",
    "    return self.solutions\n",
    "\n",
    "  def tell(self, reward):\n",
    "    # input must be a numpy float array\n",
    "    assert(len(reward) == self.popsize), \"Inconsistent reward_table size reported.\"\n",
    "\n",
    "    idx = np.argsort(reward)[::-1]\n",
    "\n",
    "    best_reward = reward[idx[0]]\n",
    "    best_mu = self.solutions[idx[0]]\n",
    "\n",
    "    self.curr_best_reward = best_reward\n",
    "    self.curr_best_mu = best_mu\n",
    "\n",
    "    if self.first_interation:\n",
    "      self.first_interation = False\n",
    "      self.best_reward = self.curr_best_reward\n",
    "      self.best_mu = best_mu\n",
    "    else:\n",
    "      if self.forget_best or (self.curr_best_reward > self.best_reward):\n",
    "        self.best_mu = best_mu\n",
    "        self.best_reward = self.curr_best_reward\n",
    "\n",
    "    # main bit:\n",
    "    # standardize the rewards to have a gaussian distribution\n",
    "    normalized_reward = (reward - np.mean(reward)) / np.std(reward)\n",
    "    self.mu += self.learning_rate/(self.popsize*self.sigma)*np.dot(self.epsilon.T, normalized_reward)\n",
    "\n",
    "    # adjust sigma according to the adaptive sigma calculation\n",
    "    if (self.sigma > self.sigma_limit):\n",
    "      self.sigma *= self.sigma_decay\n",
    "\n",
    "    if (self.learning_rate > self.learning_rate_limit):\n",
    "      self.learning_rate *= self.learning_rate_decay\n",
    "\n",
    "  def done(self):\n",
    "    return False\n",
    "\n",
    "  def current_param(self):\n",
    "    return self.curr_best_mu\n",
    "\n",
    "  def best_param(self):\n",
    "    return self.best_mu\n",
    "\n",
    "  def result(self): # return best params so far, along with historically best reward, curr reward, sigma\n",
    "    return (self.best_mu, self.best_reward, self.curr_best_reward, self.sigma)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Args = namedtuple('Args', ['batch_size', 'test_batch_size', 'epochs', 'lr', 'cuda', 'seed', 'log_interval'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "args = Args(batch_size=1000, test_batch_size=1000, epochs=30, lr=0.001, cuda=True, seed=0, log_interval=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "torch.manual_seed(args.seed)\n",
    "if args.cuda:\n",
    "  torch.cuda.manual_seed(args.seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "kwargs = {'num_workers': 1, 'pin_memory': True} if args.cuda else {}\n",
    "\n",
    "train_loader = torch.utils.data.DataLoader(\n",
    "  datasets.MNIST('MNIST_data', train=True, download=True, transform=transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))])),\n",
    "  batch_size=args.batch_size, shuffle=True, **kwargs)\n",
    "\n",
    "valid_loader = train_loader\n",
    "\n",
    "test_loader = torch.utils.data.DataLoader(\n",
    "  datasets.MNIST('MNIST_data', train=False, transform=transforms.Compose([transforms.ToTensor(),transforms.Normalize((0.1307,), (0.3081,))])),\n",
    "  batch_size=args.batch_size, shuffle=True, **kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "  def __init__(self):\n",
    "    super(Net, self).__init__()\n",
    "    self.num_filter1 = 8\n",
    "    self.num_filter2 = 16\n",
    "    self.num_padding = 2\n",
    "    # input is 28x28\n",
    "    # padding=2 for same padding\n",
    "    self.conv1 = nn.Conv2d(1, self.num_filter1, 5, padding=self.num_padding)\n",
    "    # feature map size is 14*14 by pooling\n",
    "    # padding=2 for same padding\n",
    "    self.conv2 = nn.Conv2d(self.num_filter1, self.num_filter2, 5, padding=self.num_padding)\n",
    "    # feature map size is 7*7 by pooling\n",
    "    self.fc = nn.Linear(self.num_filter2*7*7, 10)\n",
    "\n",
    "  def forward(self, x):\n",
    "    x = F.max_pool2d(F.relu(self.conv1(x)), 2)\n",
    "    x = F.max_pool2d(F.relu(self.conv2(x)), 2)\n",
    "    x = x.view(-1, self.num_filter2*7*7)   # reshape Variable\n",
    "    x = self.fc(x)\n",
    "    return F.log_softmax(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "NPOPULATION = 101\n",
    "weight_decay_coef = 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''\n",
    "models = []\n",
    "for i in range(NPOPULATION):\n",
    "  model = Net()\n",
    "  if args.cuda:\n",
    "    model.cuda()\n",
    "  model.eval()\n",
    "  models.append(model)\n",
    "'''\n",
    "\n",
    "model = Net()\n",
    "if args.cuda:\n",
    "  model.cuda()\n",
    "\n",
    "orig_model = copy.deepcopy(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11274\n"
     ]
    }
   ],
   "source": [
    "# get init params\n",
    "orig_params = []\n",
    "model_shapes = []\n",
    "for param in orig_model.parameters():\n",
    "  p = param.data.cpu().numpy()\n",
    "  model_shapes.append(p.shape)\n",
    "  orig_params.append(p.flatten())\n",
    "orig_params_flat = np.concatenate(orig_params)\n",
    "NPARAMS = len(orig_params_flat)\n",
    "print(NPARAMS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def update_model(flat_param, model, model_shapes):\n",
    "  idx = 0\n",
    "  i = 0\n",
    "  for param in model.parameters():\n",
    "    delta = np.product(model_shapes[i])\n",
    "    block = flat_param[idx:idx+delta]\n",
    "    block = np.reshape(block, model_shapes[i])\n",
    "    i += 1\n",
    "    idx += delta\n",
    "    block_data = torch.from_numpy(block).float()\n",
    "    if args.cuda:\n",
    "      block_data = block_data.cuda()\n",
    "    param.data = block_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def evaluate(model, test_loader, print_mode=True, return_loss=False):\n",
    "  model.eval()\n",
    "  test_loss = 0\n",
    "  correct = 0\n",
    "  for data, target in test_loader:\n",
    "    if args.cuda:\n",
    "      data, target = data.cuda(), target.cuda()\n",
    "    data, target = Variable(data, volatile=True), Variable(target)\n",
    "    output = model(data)\n",
    "    test_loss += F.nll_loss(output, target, size_average=False).data[0] # sum up batch loss\n",
    "    pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability\n",
    "    correct += pred.eq(target.data.view_as(pred)).cpu().sum()\n",
    "\n",
    "  test_loss /= len(test_loader.dataset)\n",
    "  acc = correct / len(test_loader.dataset)\n",
    "  \n",
    "  if print_mode:\n",
    "    print('\\nAverage loss: {:.4f}, Accuracy: {}/{} ({:.4f}%)\\n'.format(\n",
    "      test_loss, correct, len(test_loader.dataset),\n",
    "      100. * acc))\n",
    "  \n",
    "  if return_loss:\n",
    "    return test_loss\n",
    "  return acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50_w,101)-aCMA-ES (mu_w=27.2,w_1=8%) in dimension 11274 (seed=824008, Mon Aug 14 19:50:15 2017)\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "es = SimpleES(NPARAMS,\n",
    "              popsize=NPOPULATION,\n",
    "              sigma_init=0.01,\n",
    "              sigma_decay=0.999,\n",
    "              sigma_alpha=0.2,\n",
    "              sigma_limit=0.001,\n",
    "              elite_ratio=0.1,\n",
    "              average_baseline=False,\n",
    "              forget_best=True\n",
    "             )\n",
    "\"\"\"\n",
    "es = CMAES(NPARAMS, sigma_init=0.01, popsize=NPOPULATION)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def worker(procnum, model, solution, data, target, send_end):\n",
    "  update_model(solution, model, model_shapes)\n",
    "  output = model(data)\n",
    "  loss = F.nll_loss(output, target)\n",
    "  reward = - loss.data[0]\n",
    "  send_end.send(reward)\n",
    "\n",
    "def batch_simulation(model_list, solutions, data, target, process_count):\n",
    "  jobs = []\n",
    "  pipe_list = []\n",
    "\n",
    "  for i in range(process_count):\n",
    "    recv_end, send_end = mp.Pipe(False)\n",
    "    p = mp.Process(target=worker, args=(i, model_list[i], solutions[i], data, target, send_end))\n",
    "    jobs.append(p)\n",
    "    pipe_list.append(recv_end)\n",
    "\n",
    "  for p in jobs:\n",
    "    p.start()\n",
    "\n",
    "  for p in jobs:\n",
    "    p.join()\n",
    "\n",
    "  result_list = [x.recv() for x in pipe_list]\n",
    "  return np.array(result_list)\n",
    "\n",
    "\n",
    "def batch_simulation_sequential(model_list, solutions, data, target, process_count):\n",
    "  result_list = []\n",
    "  for i in range(process_count):\n",
    "    update_model(solutions[i], model_list[i], model_shapes)\n",
    "    output = model_list[i](data)\n",
    "    loss = F.nll_loss(output, target)\n",
    "    reward = - loss.data[0]\n",
    "    result_list.append(reward)\n",
    "  return np.array(result_list)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 0 -2.30188655853 -0.000104220946267 0.00997634615744\n",
      "1 5 -2.30154514313 -8.84061715779e-06 0.00986652493537\n",
      "1 10 -2.30042457581 0.000152815991869 0.0097671816071\n",
      "1 15 -2.29986000061 0.000268667177688 0.00967575199719\n",
      "1 20 -2.29250359535 0.000427199407633 0.0095904954213\n",
      "1 25 -2.28641533852 0.000454197834952 0.00951065872441\n",
      "1 30 -2.27578568459 0.000373592603108 0.00943559522572\n",
      "1 35 -2.25599741936 0.000583757802138 0.00936481786393\n",
      "1 40 -2.23393464088 0.000605808365844 0.00929836381506\n",
      "1 45 -2.20563745499 0.000878342698827 0.00923578167451\n",
      "1 50 -2.15085124969 0.00102741648315 0.0091768468182\n",
      "1 55 -2.11342811584 0.00114535202013 0.00912102638875\n",
      "valid_acc 50.94833333333333\n",
      "best valid_acc 50.94833333333333\n",
      "2 0 -2.04475855827 0.00124462044262 0.00906841210485\n",
      "2 5 -1.96643698215 0.00139885140467 0.00901885145178\n",
      "2 10 -1.92456114292 0.00150639932673 0.00897198282647\n",
      "2 15 -1.84836673737 0.0014787042054 0.00892804404344\n",
      "2 20 -1.76283228397 0.00166931656861 0.00888666918016\n",
      "2 25 -1.65202784538 0.00159684424312 0.00884761930557\n",
      "2 30 -1.5614323616 0.00176086092518 0.00881085816012\n",
      "2 35 -1.51074802876 0.00186368230732 0.00877621929025\n",
      "2 40 -1.3825763464 0.00200140552078 0.00874364684307\n",
      "2 45 -1.33094573021 0.00218381543566 0.00871340447133\n",
      "2 50 -1.25108373165 0.00216464126257 0.00868507622582\n",
      "2 55 -1.18051624298 0.00225977005697 0.00865847178507\n",
      "valid_acc 73.44166666666668\n",
      "best valid_acc 73.44166666666668\n",
      "3 0 -1.15972733498 0.00227150880845 0.00863347701892\n",
      "3 5 -1.07200860977 0.0025233022026 0.00861013650939\n",
      "3 10 -1.03631484509 0.00255583675288 0.00858839529876\n",
      "3 15 -1.02267026901 0.00252453322934 0.00856830021613\n",
      "3 20 -0.932451426983 0.00269600055034 0.00854943270675\n",
      "3 25 -0.869188427925 0.00272199953986 0.00853178643667\n",
      "3 30 -0.882929742336 0.00272199953986 0.00851539483121\n",
      "3 35 -0.873862326145 0.00289639677519 0.00850018528899\n",
      "3 40 -0.839174687862 0.00294874509058 0.00848579428436\n",
      "3 45 -0.776603341103 0.00294874509058 0.00847216898628\n",
      "3 50 -0.822721898556 0.00294874509058 0.00845940461424\n",
      "3 55 -0.720545887947 0.00302320969655 0.00844757149941\n",
      "valid_acc 80.53833333333333\n",
      "best valid_acc 80.53833333333333\n",
      "4 0 -0.751286268234 0.00302320969655 0.0084367361038\n",
      "4 5 -0.734981656075 0.00303792888535 0.008426730173\n",
      "4 10 -0.725062191486 0.00303792888535 0.00841748300598\n",
      "4 15 -0.732257306576 0.00303792888535 0.00840894015294\n",
      "4 20 -0.754565060139 0.00309445653748 0.00840070888638\n",
      "4 25 -0.65068256855 0.00311557827689 0.00839314664653\n",
      "4 30 -0.693704605103 0.00311557827689 0.00838627498061\n",
      "4 35 -0.671292901039 0.00328702661241 0.00838013551095\n",
      "4 40 -0.676898896694 0.00328702661241 0.00837432873682\n",
      "4 45 -0.65058028698 0.00332697606348 0.00836895502947\n",
      "4 50 -0.637286603451 0.00316655168739 0.00836399902584\n",
      "4 55 -0.632121384144 0.00316655168739 0.00835928877472\n",
      "valid_acc 83.31333333333333\n",
      "best valid_acc 83.31333333333333\n",
      "5 0 -0.582563757896 0.0032730868692 0.00835497816696\n",
      "5 5 -0.647633433342 0.00315001680641 0.00835100627098\n",
      "5 10 -0.64404720068 0.00315001680641 0.0083472370018\n",
      "5 15 -0.602363288403 0.00315001680641 0.00834362619502\n",
      "5 20 -0.556894898415 0.0032157705315 0.00834036252659\n",
      "5 25 -0.573666810989 0.00329401558544 0.00833746093789\n",
      "5 30 -0.592216312885 0.00329401558544 0.0083347289653\n",
      "5 35 -0.553424715996 0.00326571906129 0.00833206730485\n",
      "5 40 -0.577597796917 0.00333898569004 0.0083298127423\n",
      "5 45 -0.567076921463 0.00333898569004 0.0083279403515\n",
      "5 50 -0.548454582691 0.00333898569004 0.00832624891542\n",
      "5 55 -0.52343839407 0.00333898569004 0.00832492523069\n",
      "valid_acc 84.62333333333333\n",
      "best valid_acc 84.62333333333333\n",
      "6 0 -0.555111289024 0.00333898569004 0.00832391558462\n",
      "6 5 -0.548661112785 0.00342836538578 0.00832320634711\n",
      "6 10 -0.508737027645 0.00342836538578 0.00832293026959\n",
      "6 15 -0.516840994358 0.00342836538578 0.00832257396178\n",
      "6 20 -0.489192068577 0.00356009773054 0.00832231719472\n",
      "6 25 -0.509803771973 0.00356009773054 0.00832198692252\n",
      "6 30 -0.538019299507 0.00356009773054 0.00832155653168\n",
      "6 35 -0.544789195061 0.00356009773054 0.00832117214821\n",
      "6 40 -0.477254956961 0.00347384870701 0.00832079704851\n",
      "6 45 -0.509906172752 0.00347207625524 0.00832048097451\n",
      "6 50 -0.486025422812 0.00347207625524 0.00832029088758\n",
      "6 55 -0.488899201155 0.00354689814062 0.00832005539572\n",
      "valid_acc 85.74000000000001\n",
      "best valid_acc 85.74000000000001\n",
      "7 0 -0.4938557446 0.00356634090435 0.00832000242695\n",
      "7 5 -0.514277815819 0.00356634090435 0.00832024257952\n",
      "7 10 -0.521378874779 0.00356634090435 0.00832076501264\n",
      "7 15 -0.489254742861 0.00356634090435 0.00832130987299\n",
      "7 20 -0.495481461287 0.00356634090435 0.00832194796863\n",
      "7 25 -0.493433535099 0.00356634090435 0.00832254797361\n",
      "7 30 -0.49670112133 0.00361776550241 0.00832315876829\n",
      "7 35 -0.475669145584 0.00361776550241 0.00832384221842\n",
      "7 40 -0.44043982029 0.00361776550241 0.00832444440872\n",
      "7 45 -0.44894823432 0.00365272520692 0.00832502226976\n",
      "7 50 -0.486418634653 0.00365272520692 0.00832578597685\n",
      "7 55 -0.494576185942 0.00365272520692 0.00832672132785\n",
      "valid_acc 86.44\n",
      "best valid_acc 86.44\n",
      "8 0 -0.474964469671 0.00365272520692 0.00832744326995\n",
      "8 5 -0.481804490089 0.00365272520692 0.00832804164327\n",
      "8 10 -0.495767384768 0.00358691423194 0.00832889787554\n",
      "8 15 -0.469566315413 0.00358691423194 0.0083299742432\n",
      "8 20 -0.406027793884 0.00370390745718 0.00833104073137\n",
      "8 25 -0.4797706604 0.00370390745718 0.00833226620016\n",
      "8 30 -0.421836823225 0.00370390745718 0.0083332904699\n",
      "8 35 -0.415539801121 0.00370390745718 0.00833432108404\n",
      "8 40 -0.442087709904 0.00348948276725 0.00833544654468\n",
      "8 45 -0.443665802479 0.00348948276725 0.00833668185737\n",
      "8 50 -0.464618831873 0.00348948276725 0.00833779938571\n",
      "8 55 -0.411580204964 0.00348948276725 0.00833894045648\n",
      "valid_acc 87.195\n",
      "best valid_acc 87.195\n",
      "9 0 -0.444044321775 0.00348948276725 0.00834012976893\n",
      "9 5 -0.438996553421 0.00348948276725 0.00834105548162\n",
      "9 10 -0.396656602621 0.00345385985618 0.00834202254204\n",
      "9 15 -0.440406292677 0.00345385985618 0.00834300207104\n",
      "9 20 -0.471731871367 0.00345385985618 0.0083439073512\n",
      "9 25 -0.42530310154 0.0035404251068 0.00834469475687\n",
      "9 30 -0.426431059837 0.0035404251068 0.00834541619443\n",
      "9 35 -0.479815900326 0.0035404251068 0.00834606018618\n",
      "9 40 -0.502851605415 0.0035404251068 0.00834674065494\n",
      "9 45 -0.398344874382 0.0035404251068 0.00834755236953\n",
      "9 50 -0.393347144127 0.00344582005097 0.00834832887028\n",
      "9 55 -0.432008981705 0.00347365736642 0.00834897091672\n",
      "valid_acc 87.64999999999999\n",
      "best valid_acc 87.64999999999999\n",
      "10 0 -0.428461015224 0.00347365736642 0.0083496493093\n",
      "10 5 -0.398722678423 0.00347365736642 0.00835037483641\n",
      "10 10 -0.451933205128 0.00347365736642 0.00835131407175\n",
      "10 15 -0.467819690704 0.00347365736642 0.00835231917344\n",
      "10 20 -0.445986062288 0.00347365736642 0.00835340816212\n",
      "10 25 -0.442283987999 0.00347365736642 0.00835433347117\n",
      "10 30 -0.438314348459 0.00347365736642 0.00835548739707\n",
      "10 35 -0.428677648306 0.00347365736642 0.00835687076531\n",
      "10 40 -0.371204853058 0.00347365736642 0.0083583612874\n",
      "10 45 -0.366370081902 0.00347365736642 0.00835991986266\n",
      "10 50 -0.455912739038 0.00347365736642 0.0083615981634\n",
      "10 55 -0.453557878733 0.00347365736642 0.00836323410511\n",
      "valid_acc 88.02\n",
      "best valid_acc 88.02\n",
      "11 0 -0.452544510365 0.00347365736642 0.00836481768014\n",
      "11 5 -0.414467513561 0.00347365736642 0.00836645178134\n",
      "11 10 -0.413374871016 0.00347365736642 0.0083683556223\n",
      "11 15 -0.400757998228 0.00347365736642 0.00837010697317\n",
      "11 20 -0.414122521877 0.00347365736642 0.00837204283882\n",
      "11 25 -0.377120643854 0.00347365736642 0.00837392902288\n",
      "11 30 -0.457289218903 0.00347365736642 0.00837581996195\n",
      "11 35 -0.470564782619 0.00347365736642 0.00837805131564\n",
      "11 40 -0.382924079895 0.00347365736642 0.00838055203239\n",
      "11 45 -0.433569461107 0.00347365736642 0.00838302111799\n",
      "11 50 -0.401406645775 0.00347365736642 0.00838553299743\n",
      "11 55 -0.380652040243 0.00346441595577 0.00838794556041\n",
      "valid_acc 88.535\n",
      "best valid_acc 88.535\n",
      "12 0 -0.368129462004 0.00346441595577 0.0083903145233\n",
      "12 5 -0.516964316368 0.00346441595577 0.00839259026954\n",
      "12 10 -0.377472698689 0.00346441595577 0.00839481933377\n",
      "12 15 -0.40198096633 0.00347125438557 0.0083970350107\n",
      "12 20 -0.385153889656 0.00347125438557 0.00839920621186\n",
      "12 25 -0.38338419795 0.00347125438557 0.00840131451697\n",
      "12 30 -0.395070523024 0.00347125438557 0.00840342001963\n",
      "12 35 -0.407203435898 0.00347125438557 0.00840558078874\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12 40 -0.368787884712 0.00347125438557 0.00840779767934\n",
      "12 45 -0.399945020676 0.00347125438557 0.00840993451063\n",
      "12 50 -0.389169663191 0.00347125438557 0.00841218920854\n",
      "12 55 -0.356613248587 0.00347125438557 0.00841441264921\n",
      "valid_acc 88.92\n",
      "best valid_acc 88.92\n",
      "13 0 -0.409674197435 0.00347125438557 0.00841665073356\n",
      "13 5 -0.38014036417 0.00347125438557 0.00841904165577\n",
      "13 10 -0.38029807806 0.00347125438557 0.00842130798683\n",
      "13 15 -0.401214748621 0.00347125438557 0.00842335753393\n",
      "13 20 -0.337412327528 0.00353673239652 0.00842541998724\n",
      "13 25 -0.358988970518 0.00353673239652 0.00842752978508\n",
      "13 30 -0.385317146778 0.00353673239652 0.00842950102561\n",
      "13 35 -0.380937457085 0.00353673239652 0.0084313551979\n",
      "13 40 -0.329854160547 0.00353673239652 0.00843326451968\n",
      "13 45 -0.408561378717 0.00353673239652 0.0084351830257\n",
      "13 50 -0.385018527508 0.00353673239652 0.0084371502158\n",
      "13 55 -0.368632793427 0.00353673239652 0.00843925457767\n",
      "valid_acc 89.47666666666667\n",
      "best valid_acc 89.47666666666667\n",
      "14 0 -0.413514822721 0.00353673239652 0.00844151860289\n",
      "14 5 -0.367757558823 0.00353673239652 0.00844376798563\n",
      "14 10 -0.367182970047 0.00353673239652 0.00844597296101\n",
      "14 15 -0.337049394846 0.00353673239652 0.00844825453856\n",
      "14 20 -0.341036975384 0.00353673239652 0.00845043968377\n",
      "14 25 -0.390864491463 0.00353673239652 0.00845263878749\n",
      "14 30 -0.388759583235 0.00353673239652 0.00845463430542\n",
      "14 35 -0.372007012367 0.00353673239652 0.0084566123913\n",
      "14 40 -0.363371372223 0.00353673239652 0.00845870356374\n",
      "14 45 -0.352515608072 0.00353673239652 0.00846093966595\n",
      "14 50 -0.34367159009 0.00353673239652 0.00846314314896\n",
      "14 55 -0.390560716391 0.00353673239652 0.00846541375369\n",
      "valid_acc 89.85\n",
      "best valid_acc 89.85\n",
      "15 0 -0.335469871759 0.00353673239652 0.00846751676457\n",
      "15 5 -0.34680518508 0.00353673239652 0.00846952344064\n",
      "15 10 -0.353821694851 0.00353673239652 0.00847140769934\n",
      "15 15 -0.338808208704 0.00353673239652 0.0084732183707\n",
      "15 20 -0.382106423378 0.00353673239652 0.00847486701951\n",
      "15 25 -0.336104393005 0.00353673239652 0.00847628011869\n",
      "15 30 -0.352957397699 0.00353673239652 0.00847745757066\n",
      "15 35 -0.32245644927 0.00356208198663 0.00847857493121\n",
      "15 40 -0.298005282879 0.00371875679692 0.00847972532559\n",
      "15 45 -0.32773193717 0.00371875679692 0.00848095085446\n",
      "15 50 -0.333383351564 0.00372877959352 0.00848194287898\n",
      "15 55 -0.342106729746 0.00372877959352 0.00848300407907\n",
      "valid_acc 90.19333333333334\n",
      "best valid_acc 90.19333333333334\n",
      "16 0 -0.370067477226 0.00372877959352 0.00848416734052\n",
      "16 5 -0.353035897017 0.00372877959352 0.00848517829541\n",
      "16 10 -0.310938149691 0.00372877959352 0.00848616245161\n",
      "16 15 -0.325583189726 0.00372877959352 0.00848733147435\n",
      "16 20 -0.32462349534 0.00372877959352 0.00848874459925\n",
      "16 25 -0.353033512831 0.00372877959352 0.00849041230235\n",
      "16 30 -0.338174432516 0.00372877959352 0.0084922053747\n",
      "16 35 -0.366127073765 0.00357281743179 0.00849387273072\n",
      "16 40 -0.362642109394 0.00357281743179 0.00849576239737\n",
      "16 45 -0.364894062281 0.00357281743179 0.00849775896808\n",
      "16 50 -0.389769822359 0.00357281743179 0.0084993735765\n",
      "16 55 -0.328338831663 0.00357281743179 0.00850101649592\n",
      "valid_acc 90.58333333333334\n",
      "best valid_acc 90.58333333333334\n",
      "17 0 -0.31429669261 0.00357281743179 0.00850268076628\n",
      "17 5 -0.299549341202 0.00357281743179 0.00850435818889\n",
      "17 10 -0.326155900955 0.00357281743179 0.00850606610823\n",
      "17 15 -0.288147926331 0.00357281743179 0.00850773656925\n",
      "17 20 -0.334766060114 0.00357281743179 0.00850933438796\n",
      "17 25 -0.356620520353 0.00357281743179 0.00851078302066\n",
      "17 30 -0.315921872854 0.00357281743179 0.00851234019124\n",
      "17 35 -0.315600097179 0.00357281743179 0.00851376436701\n",
      "17 40 -0.340429782867 0.00357281743179 0.00851492784674\n",
      "17 45 -0.290505170822 0.00357281743179 0.00851608303232\n",
      "17 50 -0.288143336773 0.00357281743179 0.00851718810155\n",
      "17 55 -0.310297489166 0.00357281743179 0.00851837005421\n",
      "valid_acc 90.91333333333334\n",
      "best valid_acc 90.91333333333334\n",
      "18 0 -0.325538635254 0.00357281743179 0.00851963445167\n",
      "18 5 -0.298332452774 0.00357281743179 0.00852090495016\n",
      "18 10 -0.357591301203 0.00357281743179 0.00852214012063\n",
      "18 15 -0.318363249302 0.00357281743179 0.00852350412391\n",
      "18 20 -0.32330840826 0.00357281743179 0.00852515631734\n",
      "18 25 -0.377976536751 0.00357281743179 0.0085269422246\n",
      "18 30 -0.29754537344 0.00357281743179 0.00852865755023\n",
      "18 35 -0.335107713938 0.00357281743179 0.00853027603795\n",
      "18 40 -0.300097197294 0.00357281743179 0.00853183451858\n",
      "18 45 -0.283249437809 0.00321322315601 0.0085335346279\n",
      "18 50 -0.281575113535 0.00321322315601 0.00853516896598\n",
      "18 55 -0.335786789656 0.00321322315601 0.00853670839221\n",
      "valid_acc 91.14666666666666\n",
      "best valid_acc 91.14666666666666\n",
      "19 0 -0.297594726086 0.00321322315601 0.00853818917212\n",
      "19 5 -0.259875893593 0.00321322315601 0.00853935514249\n",
      "19 10 -0.302015393972 0.00321322315601 0.00854025639518\n",
      "19 15 -0.281935632229 0.00321322315601 0.00854110295792\n",
      "19 20 -0.327059268951 0.00289017806502 0.00854196191927\n",
      "19 25 -0.320149749517 0.00289017806502 0.00854278890449\n",
      "19 30 -0.289493262768 0.00287322981937 0.00854354199851\n",
      "19 35 -0.338016748428 0.00287322981937 0.00854432279992\n",
      "19 40 -0.310231685638 0.00287322981937 0.00854476308363\n",
      "19 45 -0.293966770172 0.00287322981937 0.00854518038764\n",
      "19 50 -0.250652343035 0.00287322981937 0.00854546944409\n",
      "19 55 -0.304200470448 0.00287322981937 0.008545524852\n",
      "valid_acc 91.685\n",
      "best valid_acc 91.685\n",
      "20 0 -0.310306996107 0.00287322981937 0.00854552450264\n",
      "20 5 -0.281434714794 0.00287322981937 0.00854567474722\n",
      "20 10 -0.303747981787 0.00287322981937 0.00854588291749\n",
      "20 15 -0.264568626881 0.00287322981937 0.00854624445256\n",
      "20 20 -0.272343605757 0.00287322981937 0.00854677949826\n",
      "20 25 -0.284742534161 0.00287322981937 0.00854751092799\n",
      "20 30 -0.278054028749 0.00287322981937 0.00854822587295\n",
      "20 35 -0.296374052763 0.00287322981937 0.00854888272658\n",
      "20 40 -0.252073436975 0.00287322981937 0.00854964242431\n",
      "20 45 -0.310188174248 0.00287322981937 0.00855047008843\n",
      "20 50 -0.292423903942 0.00287322981937 0.00855141375524\n",
      "20 55 -0.336364001036 0.00287322981937 0.00855211192797\n",
      "valid_acc 91.89833333333334\n",
      "best valid_acc 91.89833333333334\n",
      "21 0 -0.292767047882 0.00287322981937 0.00855278554276\n",
      "21 5 -0.281628072262 0.00287322981937 0.00855393189597\n",
      "21 10 -0.296804130077 0.00287322981937 0.00855536374219\n",
      "21 15 -0.30422270298 0.00287322981937 0.00855683880249\n",
      "21 20 -0.284721583128 0.00287322981937 0.00855843157275\n",
      "21 25 -0.26062142849 0.00287322981937 0.00855989467051\n",
      "21 30 -0.338274866343 0.00287322981937 0.00856127285639\n",
      "21 35 -0.232182592154 0.00285805538394 0.00856236047164\n",
      "21 40 -0.268162757158 0.00274993222452 0.00856342319299\n",
      "21 45 -0.307705640793 0.00274993222452 0.00856457400375\n",
      "21 50 -0.267096161842 0.00274993222452 0.00856586749483\n",
      "21 55 -0.234649091959 0.00274993222452 0.00856732105345\n",
      "valid_acc 92.00666666666667\n",
      "best valid_acc 92.00666666666667\n",
      "22 0 -0.263715803623 0.00270269984276 0.00856891385255\n",
      "22 5 -0.30333507061 0.00270269984276 0.00857071640168\n",
      "22 10 -0.273148834705 0.00270269984276 0.00857288884584\n",
      "22 15 -0.241772577167 0.00270269984276 0.00857546690642\n",
      "22 20 -0.259981513023 0.00270269984276 0.00857817753198\n",
      "22 25 -0.258474051952 0.00270269984276 0.00858081239486\n",
      "22 30 -0.286493718624 0.00270269984276 0.00858329379137\n",
      "22 35 -0.283040612936 0.00270269984276 0.0085856058026\n",
      "22 40 -0.245792105794 0.00270269984276 0.00858805265457\n",
      "22 45 -0.28637394309 0.00248396252044 0.00859044853159\n",
      "22 50 -0.287967324257 0.00248396252044 0.00859267827117\n",
      "22 55 -0.266022980213 0.00248396252044 0.0085949350419\n",
      "valid_acc 92.46333333333332\n",
      "best valid_acc 92.46333333333332\n",
      "23 0 -0.282053053379 0.00241904668618 0.00859707185485\n",
      "23 5 -0.265864163637 0.00241904668618 0.00859899389898\n",
      "23 10 -0.237159788609 0.00241904668618 0.00860069403106\n",
      "23 15 -0.260125666857 0.00246061243544 0.00860237522788\n",
      "23 20 -0.252279579639 0.00246061243544 0.00860404867067\n",
      "23 25 -0.244729071856 0.00246061243544 0.00860572514032\n",
      "23 30 -0.224003195763 0.00246061243544 0.00860712287057\n",
      "23 35 -0.217884942889 0.00246061243544 0.00860853414105\n",
      "23 40 -0.238868743181 0.00246061243544 0.0086100123151\n",
      "23 45 -0.250196635723 0.00246061243544 0.00861140190012\n",
      "23 50 -0.242606431246 0.00246061243544 0.00861266061225\n",
      "23 55 -0.249704927206 0.00246061243544 0.00861390497879\n",
      "valid_acc 92.94\n",
      "best valid_acc 92.94\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24 0 -0.293299794197 0.00246061243544 0.0086149970754\n",
      "24 5 -0.223264634609 0.00246061243544 0.00861584524689\n",
      "24 10 -0.24683688581 0.00246061243544 0.00861659091855\n",
      "24 15 -0.263349086046 0.00246061243544 0.00861754874178\n",
      "24 20 -0.216732054949 0.00246061243544 0.00861850325544\n",
      "24 25 -0.2348626405 0.00246061243544 0.00861953219551\n",
      "24 30 -0.234241724014 0.00246061243544 0.00862057417281\n",
      "24 35 -0.250381141901 0.00246061243544 0.00862158699939\n",
      "24 40 -0.264840096235 0.00246061243544 0.0086223955385\n",
      "24 45 -0.256871283054 0.00203111159738 0.00862309556577\n",
      "24 50 -0.273311078548 0.00203111159738 0.00862397940269\n",
      "24 55 -0.238712206483 0.00203111159738 0.00862477950742\n",
      "valid_acc 93.17999999999999\n",
      "best valid_acc 93.17999999999999\n",
      "25 0 -0.258232116699 0.00203111159738 0.00862562197467\n",
      "25 5 -0.263939857483 0.00203111159738 0.00862625183666\n",
      "25 10 -0.212182253599 0.00203111159738 0.00862666167763\n",
      "25 15 -0.226712763309 0.00203111159738 0.00862691516008\n",
      "25 20 -0.241672381759 0.00203111159738 0.00862718918281\n",
      "25 25 -0.217111572623 0.00203111159738 0.00862761305297\n",
      "25 30 -0.242502659559 0.00203111159738 0.00862839045449\n",
      "25 35 -0.196998894215 0.00203111159738 0.00862919393809\n",
      "25 40 -0.230488851666 0.00203111159738 0.00862992779432\n",
      "25 45 -0.226103395224 0.00203111159738 0.00863083399537\n",
      "25 50 -0.266290396452 0.00203111159738 0.00863153937758\n",
      "25 55 -0.187603488564 0.00216508477354 0.00863209806364\n",
      "valid_acc 93.355\n",
      "best valid_acc 93.355\n",
      "26 0 -0.238556578755 0.00216508477354 0.00863249002623\n",
      "26 5 -0.250180512667 0.00216508477354 0.00863303204427\n",
      "26 10 -0.252339452505 0.00216508477354 0.00863382326198\n",
      "26 15 -0.240922346711 0.00216508477354 0.00863443270241\n",
      "26 20 -0.233531489968 0.00216508477354 0.00863519280469\n",
      "26 25 -0.188461557031 0.00216508477354 0.00863609687437\n",
      "26 30 -0.202141880989 0.00216508477354 0.00863707297293\n",
      "26 35 -0.209444329143 0.00216508477354 0.00863815539131\n",
      "26 40 -0.219994574785 0.00216508477354 0.00863899963424\n",
      "26 45 -0.201667964458 0.00216508477354 0.0086397473553\n",
      "26 50 -0.22791403532 0.00216508477354 0.00864032871977\n",
      "26 55 -0.196904048324 0.00216508477354 0.0086409122576\n",
      "valid_acc 93.52000000000001\n",
      "best valid_acc 93.52000000000001\n",
      "27 0 -0.193548068404 0.00216508477354 0.00864136785409\n",
      "27 5 -0.222304925323 0.00216508477354 0.00864170599473\n",
      "27 10 -0.239575907588 0.00216508477354 0.00864208375698\n",
      "27 15 -0.252901792526 0.00213463383903 0.00864240467424\n",
      "27 20 -0.19518892467 0.00223342912356 0.00864287932562\n",
      "27 25 -0.212306067348 0.00237130229002 0.0086434987916\n",
      "27 30 -0.198357522488 0.00237130229002 0.00864419515015\n",
      "27 35 -0.267029613256 0.00237130229002 0.0086447521187\n",
      "27 40 -0.218942806125 0.0021854136936 0.00864513395852\n",
      "27 45 -0.222811475396 0.00223116927383 0.0086454376838\n",
      "27 50 -0.246077373624 0.00223116927383 0.0086458198058\n",
      "27 55 -0.243063092232 0.00223116927383 0.00864616634987\n",
      "valid_acc 93.66499999999999\n",
      "best valid_acc 93.66499999999999\n",
      "28 0 -0.233823090792 0.00223116927383 0.00864640257374\n",
      "28 5 -0.223799318075 0.00223116927383 0.00864666740749\n",
      "28 10 -0.183198079467 0.00223116927383 0.00864697089259\n",
      "28 15 -0.206122070551 0.00223116927383 0.00864705359153\n",
      "28 20 -0.250302910805 0.00223116927383 0.00864733760497\n",
      "28 25 -0.248119920492 0.00215382141036 0.00864777855658\n",
      "28 30 -0.193779975176 0.00215382141036 0.00864816589689\n",
      "28 35 -0.197787553072 0.00215382141036 0.00864841968056\n",
      "28 40 -0.201842039824 0.00215382141036 0.00864865537051\n",
      "28 45 -0.215452775359 0.00215382141036 0.00864865725402\n",
      "28 50 -0.163478076458 0.00215382141036 0.00864850983748\n",
      "28 55 -0.213183730841 0.00215382141036 0.0086486946106\n",
      "valid_acc 93.94\n",
      "best valid_acc 93.94\n",
      "29 0 -0.174226358533 0.00215382141036 0.00864896377714\n",
      "29 5 -0.184727177024 0.00215382141036 0.00864923224695\n",
      "29 10 -0.24933539331 0.00215382141036 0.00864953241798\n",
      "29 15 -0.179790601134 0.00215382141036 0.0086498588316\n",
      "29 20 -0.194959819317 0.00240747316445 0.00865020066819\n",
      "29 25 -0.249078780413 0.00240747316445 0.00865058158287\n",
      "29 30 -0.19398035109 0.00240747316445 0.00865082814878\n",
      "29 35 -0.225033178926 0.00240747316445 0.00865112603452\n",
      "29 40 -0.209420979023 0.00235051222365 0.00865150489834\n",
      "29 45 -0.221699893475 0.00235051222365 0.00865195434766\n",
      "29 50 -0.257632166147 0.00235051222365 0.00865243750807\n",
      "29 55 -0.206783026457 0.00235051222365 0.00865271234458\n",
      "valid_acc 94.22833333333334\n",
      "best valid_acc 94.22833333333334\n",
      "30 0 -0.183102756739 0.00235051222365 0.00865305663792\n",
      "30 5 -0.234323576093 0.00235051222365 0.00865333900389\n",
      "30 10 -0.191914275289 0.00235051222365 0.00865357680644\n",
      "30 15 -0.245638370514 0.00235051222365 0.00865381458963\n",
      "30 20 -0.235824152827 0.00235051222365 0.00865398049271\n",
      "30 25 -0.25862044096 0.00235051222365 0.00865412338489\n",
      "30 30 -0.173205778003 0.00235051222365 0.00865410343285\n",
      "30 35 -0.188490569592 0.00235051222365 0.00865396865875\n",
      "30 40 -0.211396604776 0.00235051222365 0.00865366230892\n",
      "30 45 -0.225064754486 0.00235051222365 0.00865328514795\n",
      "30 50 -0.202478304505 0.00280196007007 0.00865290393407\n",
      "30 55 -0.148933574557 0.00280196007007 0.00865259586934\n",
      "valid_acc 94.22666666666667\n",
      "31 0 -0.192323297262 0.00280196007007 0.00865215285306\n",
      "31 5 -0.252190291882 0.00280196007007 0.00865169628164\n",
      "31 10 -0.197430670261 0.00280196007007 0.00865136856348\n",
      "31 15 -0.202869385481 0.00280196007007 0.00865127916414\n",
      "31 20 -0.183074533939 0.00280196007007 0.00865115193241\n",
      "31 25 -0.155473157763 0.00280196007007 0.00865088876137\n",
      "31 30 -0.204556375742 0.00280196007007 0.00865052326127\n",
      "31 35 -0.216406792402 0.00280196007007 0.00865047248973\n",
      "31 40 -0.190096959472 0.00280196007007 0.00865050090934\n",
      "31 45 -0.208175718784 0.00280196007007 0.00865058139692\n",
      "31 50 -0.20830348134 0.00280196007007 0.00865076596331\n",
      "31 55 -0.189232096076 0.00280196007007 0.00865052728674\n",
      "valid_acc 94.395\n",
      "best valid_acc 94.395\n",
      "32 0 -0.21149033308 0.00280196007007 0.0086501113063\n",
      "32 5 -0.211857169867 0.00280196007007 0.00864978370043\n",
      "32 10 -0.156882196665 0.00280196007007 0.00864972238236\n",
      "32 15 -0.216061428189 0.00280196007007 0.00864976986212\n",
      "32 20 -0.175217404962 0.00280196007007 0.00864977127208\n",
      "32 25 -0.202522069216 0.00280196007007 0.00864982027173\n",
      "32 30 -0.189804673195 0.00280196007007 0.00864962957439\n",
      "32 35 -0.173912733793 0.00280196007007 0.00864935780058\n",
      "32 40 -0.205209672451 0.00280196007007 0.00864922062216\n",
      "32 45 -0.194510161877 0.00280196007007 0.00864898150112\n",
      "32 50 -0.189443543553 0.00280196007007 0.00864840919807\n",
      "32 55 -0.206985995173 0.00280196007007 0.0086479412568\n",
      "valid_acc 94.63666666666667\n",
      "best valid_acc 94.63666666666667\n",
      "33 0 -0.16917720437 0.00280196007007 0.00864774750151\n",
      "33 5 -0.192507565022 0.00280196007007 0.00864748076389\n",
      "33 10 -0.167023226619 0.00280196007007 0.00864715013219\n",
      "33 15 -0.164945885539 0.00280196007007 0.00864679257257\n",
      "33 20 -0.195287227631 0.00280196007007 0.00864660020765\n",
      "33 25 -0.168411165476 0.00280196007007 0.00864662816168\n",
      "33 30 -0.20186060667 0.00280196007007 0.00864661135485\n",
      "33 35 -0.189507991076 0.00280196007007 0.00864643366795\n",
      "33 40 -0.192749246955 0.00280196007007 0.0086460508057\n",
      "33 45 -0.162960395217 0.00280196007007 0.00864581186137\n",
      "33 50 -0.210935428739 0.00280196007007 0.00864559076031\n",
      "33 55 -0.186730444431 0.00280196007007 0.00864535661021\n",
      "valid_acc 94.75166666666667\n",
      "best valid_acc 94.75166666666667\n",
      "34 0 -0.181135699153 0.00280196007007 0.00864500177076\n",
      "34 5 -0.156299740076 0.00280196007007 0.00864439988879\n",
      "34 10 -0.191099181771 0.00280196007007 0.00864367854109\n",
      "34 15 -0.20714956522 0.00280196007007 0.00864296816905\n",
      "34 20 -0.193437382579 0.00280196007007 0.00864235158198\n",
      "34 25 -0.15529280901 0.00280196007007 0.00864169643214\n",
      "34 30 -0.203543260694 0.00280196007007 0.00864102991304\n",
      "34 35 -0.161710739136 0.00280196007007 0.0086405122849\n",
      "34 40 -0.166770368814 0.00280196007007 0.00864030866098\n",
      "34 45 -0.183287844062 0.00280196007007 0.00864017224675\n",
      "34 50 -0.167139098048 0.00280196007007 0.00864007187424\n",
      "34 55 -0.150687634945 0.00280196007007 0.00863986391789\n",
      "valid_acc 94.90166666666666\n",
      "best valid_acc 94.90166666666666\n",
      "35 0 -0.193057596684 0.00280196007007 0.00863963762886\n",
      "35 5 -0.182796940207 0.00271467467095 0.008639284296\n",
      "35 10 -0.188187196851 0.00271467467095 0.00863888735498\n",
      "35 15 -0.152629673481 0.00271467467095 0.00863843495328\n",
      "35 20 -0.181120157242 0.00271467467095 0.00863790638142\n",
      "35 25 -0.18493501842 0.00271467467095 0.00863739768629\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "35 30 -0.149749964476 0.00271467467095 0.00863684185053\n",
      "35 35 -0.159742504358 0.00271467467095 0.00863615819735\n",
      "35 40 -0.128976076841 0.00271467467095 0.00863550936104\n",
      "35 45 -0.15264955163 0.00271467467095 0.00863499683152\n",
      "35 50 -0.164067521691 0.00271467467095 0.00863475798371\n",
      "35 55 -0.144628629088 0.00271467467095 0.00863449360725\n",
      "valid_acc 94.96166666666667\n",
      "best valid_acc 94.96166666666667\n",
      "36 0 -0.188382431865 0.00271467467095 0.00863445284221\n",
      "36 5 -0.159257903695 0.00271467467095 0.00863468149396\n",
      "36 10 -0.161554083228 0.00271467467095 0.00863509343285\n",
      "36 15 -0.197679921985 0.00271467467095 0.00863566499484\n",
      "36 20 -0.192334368825 0.00289385710066 0.00863650004802\n",
      "36 25 -0.143461883068 0.00289385710066 0.00863728846463\n",
      "36 30 -0.190130904317 0.00289385710066 0.00863800554381\n",
      "36 35 -0.137236729264 0.00289385710066 0.00863867483866\n",
      "36 40 -0.175755679607 0.00289385710066 0.00863931150323\n",
      "36 45 -0.224015012383 0.00289385710066 0.00864002938701\n",
      "36 50 -0.155479073524 0.00289385710066 0.00864051575892\n",
      "36 55 -0.205543920398 0.00289385710066 0.00864100232637\n",
      "valid_acc 95.045\n",
      "best valid_acc 95.045\n",
      "37 0 -0.171777278185 0.00289385710066 0.00864109056516\n",
      "37 5 -0.16914229095 0.00289385710066 0.00864112515589\n",
      "37 10 -0.14623093605 0.00289385710066 0.00864113237167\n",
      "37 15 -0.158630594611 0.00289385710066 0.00864118672109\n",
      "37 20 -0.170526489615 0.00289385710066 0.00864122029683\n",
      "37 25 -0.157255709171 0.00289385710066 0.00864105801929\n",
      "37 30 -0.160665497184 0.00289385710066 0.00864089445581\n",
      "37 35 -0.171335399151 0.00289385710066 0.00864053043543\n",
      "37 40 -0.113589517772 0.00280955717183 0.00864017867841\n",
      "37 45 -0.233674183488 0.00280955717183 0.00864000919339\n",
      "37 50 -0.222302049398 0.00280955717183 0.00863963008407\n",
      "37 55 -0.194561392069 0.00280955717183 0.00863920269556\n",
      "valid_acc 95.15166666666667\n",
      "best valid_acc 95.15166666666667\n",
      "38 0 -0.193203270435 0.00280955717183 0.00863886902544\n",
      "38 5 -0.154889777303 0.00280955717183 0.0086386948887\n",
      "38 10 -0.152776360512 0.00280955717183 0.0086385932103\n",
      "38 15 -0.14285607636 0.00280955717183 0.00863852492497\n",
      "38 20 -0.152134642005 0.00280955717183 0.0086385502022\n",
      "38 25 -0.194892466068 0.00280955717183 0.00863850876465\n",
      "38 30 -0.189961448312 0.00280955717183 0.00863851793868\n",
      "38 35 -0.128149196506 0.00257030823817 0.00863846618722\n",
      "38 40 -0.113957196474 0.00257030823817 0.00863837883387\n",
      "38 45 -0.194039419293 0.00257030823817 0.00863790110092\n",
      "38 50 -0.177301302552 0.00257030823817 0.00863719829363\n",
      "38 55 -0.185926958919 0.00257030823817 0.00863644699459\n",
      "valid_acc 95.25333333333333\n",
      "best valid_acc 95.25333333333333\n",
      "39 0 -0.154270395637 0.00257030823817 0.0086357158206\n",
      "39 5 -0.129710018635 0.00257030823817 0.00863491433956\n",
      "39 10 -0.183026000857 0.00257030823817 0.00863404036969\n",
      "39 15 -0.175399333239 0.00257030823817 0.00863311122205\n",
      "39 20 -0.153478950262 0.00257030823817 0.00863243286669\n",
      "39 25 -0.162343844771 0.00257030823817 0.00863214175131\n",
      "39 30 -0.132375702262 0.00257030823817 0.00863218776715\n",
      "39 35 -0.148759111762 0.00257030823817 0.00863215121902\n",
      "39 40 -0.193759068847 0.00257030823817 0.00863207960333\n",
      "39 45 -0.16574190557 0.00257030823817 0.0086320569294\n",
      "39 50 -0.169330462813 0.00257030823817 0.00863212682247\n",
      "39 55 -0.150819808245 0.00257030823817 0.00863231022351\n",
      "valid_acc 95.39\n",
      "best valid_acc 95.39\n",
      "40 0 -0.154486998916 0.00257030823817 0.00863251809335\n",
      "40 5 -0.155210196972 0.00257030823817 0.0086329363452\n",
      "40 10 -0.126649945974 0.00257030823817 0.00863325462041\n",
      "40 15 -0.163422182202 0.00257030823817 0.00863360156921\n",
      "40 20 -0.149677887559 0.00257030823817 0.00863400930865\n",
      "40 25 -0.125909462571 0.00257030823817 0.00863448090232\n",
      "40 30 -0.146648466587 0.00257030823817 0.0086347217131\n",
      "40 35 -0.140393331647 0.00262931233255 0.0086348914571\n",
      "40 40 -0.174278661609 0.00262931233255 0.00863509646658\n",
      "40 45 -0.115907728672 0.00262931233255 0.00863557598967\n",
      "40 50 -0.143963232636 0.00262931233255 0.00863619630776\n",
      "40 55 -0.175931081176 0.00262931233255 0.00863690828167\n",
      "valid_acc 95.46833333333333\n",
      "best valid_acc 95.46833333333333\n",
      "41 0 -0.148692667484 0.00262931233255 0.008637537072\n",
      "41 5 -0.151200085878 0.00262931233255 0.00863802468178\n",
      "41 10 -0.092475079 0.00266663181333 0.00863867256347\n",
      "41 15 -0.132609918714 0.00266663181333 0.00863944130573\n",
      "41 20 -0.159529909492 0.00266663181333 0.00864038357975\n",
      "41 25 -0.188801914454 0.00266663181333 0.0086411805622\n",
      "41 30 -0.138800352812 0.00266663181333 0.00864173715805\n",
      "41 35 -0.170268967748 0.00266663181333 0.00864230475335\n",
      "41 40 -0.177423998713 0.00266663181333 0.00864297908151\n",
      "41 45 -0.146524012089 0.00266663181333 0.00864360709824\n",
      "41 50 -0.173844352365 0.00266663181333 0.00864395201218\n",
      "41 55 -0.128718346357 0.00266663181333 0.00864426116543\n",
      "valid_acc 95.685\n",
      "best valid_acc 95.685\n",
      "42 0 -0.170431509614 0.00266663181333 0.00864488280433\n",
      "42 5 -0.138891085982 0.00266663181333 0.00864568382816\n",
      "42 10 -0.161592125893 0.00266663181333 0.00864657457942\n",
      "42 15 -0.152466252446 0.00266663181333 0.00864759847022\n",
      "42 20 -0.141279488802 0.00266663181333 0.00864866066145\n",
      "42 25 -0.119286313653 0.00266663181333 0.00864985375583\n",
      "42 30 -0.16092531383 0.00266663181333 0.00865102424592\n",
      "42 35 -0.161015853286 0.00266663181333 0.00865232538203\n",
      "42 40 -0.134785592556 0.00266663181333 0.00865330435111\n",
      "42 45 -0.136042922735 0.00266663181333 0.0086540869906\n",
      "42 50 -0.136851072311 0.00266663181333 0.00865481925466\n",
      "42 55 -0.133913949132 0.00266663181333 0.00865537612084\n",
      "valid_acc 95.66\n",
      "43 0 -0.105996869504 0.00266663181333 0.00865562155955\n",
      "43 5 -0.130851045251 0.00266663181333 0.00865557596082\n",
      "43 10 -0.13103929162 0.00266663181333 0.00865519396252\n",
      "43 15 -0.127994313836 0.00266663181333 0.00865474815273\n",
      "43 20 -0.160183414817 0.00266663181333 0.00865450071012\n",
      "43 25 -0.131144657731 0.00266663181333 0.00865444886518\n",
      "43 30 -0.175071582198 0.00266663181333 0.008654293975\n",
      "43 35 -0.202515378594 0.00266663181333 0.00865421206572\n",
      "43 40 -0.144317641854 0.00266663181333 0.00865410299027\n",
      "43 45 -0.141643851995 0.00266663181333 0.00865396561897\n",
      "43 50 -0.157562270761 0.00266663181333 0.00865374991235\n",
      "43 55 -0.112354151905 0.00266663181333 0.00865352515801\n",
      "valid_acc 95.745\n",
      "best valid_acc 95.745\n",
      "44 0 -0.115702167153 0.00266663181333 0.00865337505687\n",
      "44 5 -0.139956086874 0.00266663181333 0.00865320057545\n",
      "44 10 -0.137955904007 0.00266663181333 0.00865293887925\n",
      "44 15 -0.124025329947 0.00266663181333 0.00865261827474\n",
      "44 20 -0.160484865308 0.00266663181333 0.00865218896054\n",
      "44 25 -0.165455713868 0.00266663181333 0.0086515747574\n",
      "44 30 -0.144515991211 0.00266663181333 0.00865104193914\n",
      "44 35 -0.151586443186 0.00266663181333 0.00865043743994\n",
      "44 40 -0.132693707943 0.00266663181333 0.00864956409475\n",
      "44 45 -0.146165668964 0.00266663181333 0.00864848530238\n",
      "44 50 -0.161649078131 0.00266663181333 0.00864736759297\n",
      "44 55 -0.15406191349 0.00266663181333 0.00864640914762\n",
      "valid_acc 95.81333333333333\n",
      "best valid_acc 95.81333333333333\n",
      "45 0 -0.14632345736 0.00266663181333 0.00864553263327\n",
      "45 5 -0.150361359119 0.00266663181333 0.00864488779734\n",
      "45 10 -0.147351458669 0.00266663181333 0.00864447948502\n",
      "45 15 -0.137846529484 0.00266663181333 0.00864397852933\n",
      "45 20 -0.150102734566 0.00266663181333 0.00864318961673\n",
      "45 25 -0.151954203844 0.00266663181333 0.00864270056227\n",
      "45 30 -0.134359955788 0.00266663181333 0.00864231352224\n",
      "45 35 -0.138609305024 0.00266663181333 0.0086421535631\n",
      "45 40 -0.143971607089 0.00266663181333 0.0086419193278\n",
      "45 45 -0.140481650829 0.00266663181333 0.00864181469759\n",
      "45 50 -0.140085220337 0.00266663181333 0.00864183889854\n",
      "45 55 -0.132979854941 0.00266663181333 0.0086417041268\n",
      "valid_acc 95.96166666666667\n",
      "best valid_acc 95.96166666666667\n",
      "46 0 -0.139100015163 0.00266663181333 0.008641301477\n",
      "46 5 -0.110296539962 0.00266663181333 0.00864107476289\n",
      "46 10 -0.164128020406 0.00266663181333 0.00864101476024\n",
      "46 15 -0.122711770236 0.00266663181333 0.00864089127567\n",
      "46 20 -0.156463831663 0.00266663181333 0.00864061398444\n",
      "46 25 -0.169509291649 0.00266663181333 0.0086403041979\n",
      "46 30 -0.161614760756 0.00266663181333 0.00863991241108\n",
      "46 35 -0.145481333137 0.00266663181333 0.00863959650138\n",
      "46 40 -0.151784077287 0.00266663181333 0.00863931444671\n",
      "46 45 -0.141335502267 0.00266663181333 0.00863904953664\n",
      "46 50 -0.150985121727 0.00266663181333 0.00863894829767\n",
      "46 55 -0.0992359742522 0.00266663181333 0.00863900731387\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_acc 95.905\n",
      "47 0 -0.133553355932 0.00266663181333 0.00863888627134\n",
      "47 5 -0.135017186403 0.00266663181333 0.00863865360115\n",
      "47 10 -0.120845779777 0.00266663181333 0.00863849217871\n",
      "47 15 -0.127971023321 0.00266663181333 0.00863837726356\n",
      "47 20 -0.160439863801 0.00266663181333 0.00863840834489\n",
      "47 25 -0.122789308429 0.00266663181333 0.00863831270677\n",
      "47 30 -0.143059760332 0.00175738811011 0.0086381547545\n",
      "47 35 -0.174086123705 0.00175738811011 0.0086382779807\n",
      "47 40 -0.102100208402 0.00175738811011 0.00863853918736\n",
      "47 45 -0.112652048469 0.00175738811011 0.00863889734768\n",
      "47 50 -0.12760232389 0.00175738811011 0.00863915612046\n",
      "47 55 -0.123700983822 0.00175738811011 0.00863933362564\n",
      "valid_acc 96.015\n",
      "best valid_acc 96.015\n",
      "48 0 -0.104921407998 0.00175738811011 0.00863965190557\n",
      "48 5 -0.132464423776 0.00175738811011 0.00864006663413\n",
      "48 10 -0.134943589568 0.00175738811011 0.00864051120807\n",
      "48 15 -0.125684767962 0.00175738811011 0.00864104548426\n",
      "48 20 -0.0977127552032 0.00175738811011 0.00864161751833\n",
      "48 25 -0.116942293942 0.00175738811011 0.00864223672328\n",
      "48 30 -0.124416261911 0.00175738811011 0.00864307420628\n",
      "48 35 -0.133062526584 0.00175738811011 0.00864383302539\n",
      "48 40 -0.139567613602 0.00175738811011 0.00864448999249\n",
      "48 45 -0.115379035473 0.00175738811011 0.00864495749355\n",
      "48 50 -0.132349863648 0.00175738811011 0.00864533254714\n",
      "48 55 -0.151438489556 0.00175738811011 0.00864551180972\n",
      "valid_acc 95.97333333333333\n",
      "49 0 -0.144154593349 0.00175738811011 0.00864569831732\n",
      "49 5 -0.115349687636 0.00175738811011 0.00864581471031\n",
      "49 10 -0.118037618697 0.00175738811011 0.00864612690881\n",
      "49 15 -0.116061657667 0.00175738811011 0.00864650900252\n",
      "49 20 -0.128674715757 0.00175738811011 0.00864688405127\n",
      "49 25 -0.140754759312 0.00175738811011 0.00864700565478\n",
      "49 30 -0.16025622189 0.00175738811011 0.00864695634591\n",
      "49 35 -0.150780811906 0.00175738811011 0.00864676862271\n",
      "49 40 -0.114307224751 0.00175738811011 0.00864658554923\n",
      "49 45 -0.126923754811 0.00175738811011 0.00864623840911\n",
      "49 50 -0.141973048449 0.00175738811011 0.00864576288604\n",
      "49 55 -0.129087537527 0.00175738811011 0.00864534263489\n",
      "valid_acc 96.20833333333333\n",
      "best valid_acc 96.20833333333333\n",
      "50 0 -0.121760465205 0.00175738811011 0.00864515408118\n",
      "50 5 -0.132214337587 0.00175738811011 0.00864511550226\n",
      "50 10 -0.121924139559 0.00175738811011 0.00864511799991\n",
      "50 15 -0.144036725163 0.00175738811011 0.00864513064123\n",
      "50 20 -0.127293601632 0.00175738811011 0.0086451165728\n",
      "50 25 -0.113753207028 0.00175738811011 0.0086453914102\n",
      "50 30 -0.179563969374 0.00175738811011 0.00864559958426\n",
      "50 35 -0.102397337556 0.00175738811011 0.0086456349657\n",
      "50 40 -0.148669958115 0.00175738811011 0.00864575431597\n",
      "50 45 -0.143606796861 0.00175738811011 0.00864589144095\n",
      "50 50 -0.121339216828 0.00175738811011 0.0086457725485\n",
      "50 55 -0.115875989199 0.00175738811011 0.00864567699308\n",
      "valid_acc 96.20833333333333\n",
      "best valid_acc 96.20833333333333\n",
      "51 0 -0.106054157019 0.00175738811011 0.00864572307869\n",
      "51 5 -0.157514005899 0.00175738811011 0.00864585246427\n",
      "51 10 -0.142076209188 0.00175738811011 0.00864602434496\n",
      "51 15 -0.139497578144 0.00175738811011 0.0086462356841\n",
      "51 20 -0.113624319434 0.00175738811011 0.00864676020861\n",
      "51 25 -0.164034068584 0.00175738811011 0.0086471256656\n",
      "51 30 -0.105215184391 0.00175738811011 0.00864738249823\n",
      "51 35 -0.126910358667 0.00175738811011 0.00864767068853\n",
      "51 40 -0.097101829946 0.00175738811011 0.0086479569996\n",
      "51 45 -0.123442158103 0.00175738811011 0.00864822888743\n",
      "51 50 -0.131729856133 0.00175738811011 0.00864850106536\n",
      "51 55 -0.138627171516 0.00175738811011 0.00864876082522\n",
      "valid_acc 96.295\n",
      "best valid_acc 96.295\n",
      "52 0 -0.143563568592 0.00175738811011 0.00864883542639\n",
      "52 5 -0.106968007982 0.00175738811011 0.00864894577054\n",
      "52 10 -0.114443719387 0.00175738811011 0.008649187194\n",
      "52 15 -0.0970499962568 0.00175738811011 0.00864928076909\n",
      "52 20 -0.142725855112 0.00175738811011 0.00864910165326\n",
      "52 25 -0.156753912568 0.00175738811011 0.00864882874396\n",
      "52 30 -0.122848816216 0.00175738811011 0.00864857580815\n",
      "52 35 -0.112490944564 0.00175738811011 0.00864842691548\n",
      "52 40 -0.132544994354 0.00175738811011 0.00864840402864\n",
      "52 45 -0.118547521532 0.00175738811011 0.00864836628064\n",
      "52 50 -0.14650696516 0.00175738811011 0.00864830669561\n",
      "52 55 -0.11262743175 0.00175738811011 0.00864826378773\n",
      "valid_acc 96.39666666666666\n",
      "best valid_acc 96.39666666666666\n",
      "53 0 -0.100736990571 0.00175738811011 0.00864829808076\n",
      "53 5 -0.135274961591 0.00175738811011 0.00864830108819\n",
      "53 10 -0.115443043411 0.00175738811011 0.00864824203146\n",
      "53 15 -0.131946921349 0.00175738811011 0.00864811911093\n",
      "53 20 -0.121593609452 0.00175738811011 0.00864806654112\n",
      "53 25 -0.124704323709 0.00175738811011 0.00864799939888\n",
      "53 30 -0.117960229516 0.00175738811011 0.0086481488671\n",
      "53 35 -0.109775550663 0.00175738811011 0.00864816075556\n",
      "53 40 -0.104759559035 0.00175738811011 0.00864820110682\n",
      "53 45 -0.141387268901 0.00175738811011 0.00864837577293\n",
      "53 50 -0.13434921205 0.00175738811011 0.00864846759849\n",
      "53 55 -0.160677179694 0.00175738811011 0.00864843102518\n",
      "valid_acc 96.47166666666666\n",
      "best valid_acc 96.47166666666666\n",
      "54 0 -0.116378985345 0.00175738811011 0.00864814359141\n",
      "54 5 -0.112490169704 0.00175738811011 0.00864762311678\n",
      "54 10 -0.0990828350186 0.00175738811011 0.00864704948434\n",
      "54 15 -0.118822686374 0.00175738811011 0.00864648223792\n",
      "54 20 -0.12899595499 0.00175738811011 0.00864590295026\n",
      "54 25 -0.134126082063 0.00175738811011 0.00864532412076\n",
      "54 30 -0.123972825706 0.00175738811011 0.00864464986142\n",
      "54 35 -0.11580145359 0.00175738811011 0.00864388659576\n",
      "54 40 -0.108338147402 0.00175738811011 0.00864295184535\n",
      "54 45 -0.124578244984 0.00175738811011 0.00864210902402\n",
      "54 50 -0.107867926359 0.00175738811011 0.00864135411925\n",
      "54 55 -0.12348690629 0.00221729185673 0.00864065063459\n",
      "valid_acc 96.49833333333333\n",
      "best valid_acc 96.49833333333333\n",
      "55 0 -0.108856528997 0.00221729185673 0.00864004123224\n",
      "55 5 -0.116310350597 0.00221729185673 0.00863947040477\n",
      "55 10 -0.11868891865 0.00221729185673 0.00863895334989\n",
      "55 15 -0.118886388838 0.00221729185673 0.0086385107484\n",
      "55 20 -0.1265642941 0.00221729185673 0.00863801536614\n",
      "55 25 -0.0839126333594 0.00221729185673 0.00863737792227\n",
      "55 30 -0.13403365016 0.00221729185673 0.00863677728292\n",
      "55 35 -0.101896315813 0.00221729185673 0.00863607064767\n",
      "55 40 -0.116756550968 0.00221729185673 0.00863524658396\n",
      "55 45 -0.0983141064644 0.00221729185673 0.00863456883023\n",
      "55 50 -0.139336168766 0.00221729185673 0.00863388512924\n",
      "55 55 -0.106738090515 0.00221729185673 0.00863337239416\n",
      "valid_acc 96.50333333333333\n",
      "best valid_acc 96.50333333333333\n",
      "56 0 -0.114498540759 0.00221729185673 0.00863274112201\n",
      "56 5 -0.117132872343 0.00221729185673 0.00863212092891\n",
      "56 10 -0.111506991088 0.00221729185673 0.00863145882891\n",
      "56 15 -0.132138341665 0.00221729185673 0.00863089764688\n",
      "56 20 -0.131476685405 0.00221729185673 0.00863034388665\n",
      "56 25 -0.110383495688 0.00221729185673 0.00862965313645\n",
      "56 30 -0.112648449838 0.0021122790016 0.00862889275836\n",
      "56 35 -0.139794439077 0.0021122790016 0.00862808529261\n",
      "56 40 -0.118257507682 0.0021122790016 0.00862708073587\n",
      "56 45 -0.0961394757032 0.0021122790016 0.00862598495139\n",
      "56 50 -0.122811853886 0.0021122790016 0.00862487407869\n",
      "56 55 -0.136689856648 0.0021122790016 0.00862370332392\n",
      "valid_acc 96.52833333333334\n",
      "best valid_acc 96.52833333333334\n",
      "57 0 -0.125136628747 0.0021122790016 0.00862250943402\n",
      "57 5 -0.112782776356 0.0021122790016 0.00862132073368\n",
      "57 10 -0.101400345564 0.0021122790016 0.00862021376559\n",
      "57 15 -0.0782061815262 0.0021122790016 0.00861916582748\n",
      "57 20 -0.109341055155 0.0021122790016 0.00861808500591\n",
      "57 25 -0.11783477664 0.0021122790016 0.00861693836357\n",
      "57 30 -0.0845779776573 0.0021122790016 0.00861570200531\n",
      "57 35 -0.0851027145982 0.0021122790016 0.00861451070821\n",
      "57 40 -0.133187994361 0.0021122790016 0.00861345366612\n",
      "57 45 -0.160106480122 0.0021122790016 0.00861254205169\n",
      "57 50 -0.109331838787 0.0021122790016 0.00861163136497\n",
      "57 55 -0.14415538311 0.0021122790016 0.0086110046077\n",
      "valid_acc 96.60166666666666\n",
      "best valid_acc 96.60166666666666\n",
      "58 0 -0.109325274825 0.0021122790016 0.00861036261395\n",
      "58 5 -0.137534931302 0.0021122790016 0.00860980688201\n",
      "58 10 -0.0829112827778 0.0021122790016 0.00860934393435\n",
      "58 15 -0.117928691208 0.0021122790016 0.00860867302573\n",
      "58 20 -0.113916829228 0.0021122790016 0.00860801739512\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "58 25 -0.112217515707 0.0021122790016 0.00860729387172\n",
      "58 30 -0.104678533971 0.0021122790016 0.00860655993027\n",
      "58 35 -0.114505372941 0.0021122790016 0.0086060757498\n",
      "58 40 -0.0942439287901 0.0021122790016 0.00860566882003\n",
      "58 45 -0.111011914909 0.0021122790016 0.00860544877903\n",
      "58 50 -0.105483844876 0.0021122790016 0.00860525561413\n",
      "58 55 -0.123141109943 0.0021122790016 0.00860511221953\n",
      "valid_acc 96.63333333333334\n",
      "best valid_acc 96.63333333333334\n",
      "59 0 -0.11253798008 0.0021122790016 0.00860499113579\n",
      "59 5 -0.159926474094 0.0021122790016 0.00860486730035\n",
      "59 10 -0.112486928701 0.0021122790016 0.0086045752407\n",
      "59 15 -0.097209610045 0.0021122790016 0.00860438057331\n",
      "59 20 -0.117390230298 0.0021122790016 0.00860405933947\n",
      "59 25 -0.0937673673034 0.0021122790016 0.0086036908422\n",
      "59 30 -0.105257540941 0.0021122790016 0.00860336573475\n",
      "59 35 -0.113285526633 0.0021122790016 0.00860309709535\n",
      "59 40 -0.102194152772 0.0021122790016 0.00860297489577\n",
      "59 45 -0.119906328619 0.0021122790016 0.00860294676034\n",
      "59 50 -0.113956287503 0.0021122790016 0.00860309590036\n",
      "59 55 -0.113055117428 0.0021122790016 0.00860332358589\n",
      "valid_acc 96.665\n",
      "best valid_acc 96.665\n",
      "60 0 -0.088568277657 0.0021122790016 0.00860350513396\n",
      "60 5 -0.112056694925 0.0021122790016 0.00860369520058\n",
      "60 10 -0.118095122278 0.0021122790016 0.0086037232246\n",
      "60 15 -0.0968980491161 0.0021122790016 0.00860367680757\n",
      "60 20 -0.122945211828 0.0021122790016 0.00860350874442\n",
      "60 25 -0.124251104891 0.0021122790016 0.00860329960236\n",
      "60 30 -0.0998888686299 0.0021122790016 0.00860327842763\n",
      "60 35 -0.107460968196 0.0021122790016 0.00860333973782\n",
      "60 40 -0.0885516405106 0.0021122790016 0.00860348016598\n",
      "60 45 -0.116896674037 0.0021122790016 0.00860363262679\n",
      "60 50 -0.106308609247 0.0021122790016 0.00860383273493\n",
      "60 55 -0.0991765707731 0.0021122790016 0.00860417287922\n",
      "valid_acc 96.71333333333332\n",
      "best valid_acc 96.71333333333332\n",
      "61 0 -0.0902574285865 0.0021122790016 0.00860457649712\n",
      "61 5 -0.150825709105 0.0021122790016 0.00860469512997\n",
      "61 10 -0.115957267582 0.0021122790016 0.00860452590373\n",
      "61 15 -0.123210772872 0.0021122790016 0.00860399865635\n",
      "61 20 -0.0978501886129 0.0021122790016 0.00860357597641\n",
      "61 25 -0.131389692426 0.0021122790016 0.00860337346969\n",
      "61 30 -0.071112215519 0.0021122790016 0.008603277236\n",
      "61 35 -0.145901873708 0.0021122790016 0.00860325278858\n",
      "61 40 -0.0889560580254 0.0021122790016 0.00860322381061\n",
      "61 45 -0.112630061805 0.0021122790016 0.00860312744564\n",
      "61 50 -0.132043257356 0.0021122790016 0.00860309143414\n",
      "61 55 -0.116417638958 0.00170347309227 0.0086030046887\n",
      "valid_acc 96.765\n",
      "best valid_acc 96.765\n",
      "62 0 -0.0972819551826 0.00170347309227 0.00860288546429\n",
      "62 5 -0.0792502388358 0.00170347309227 0.00860279742073\n",
      "62 10 -0.0921157151461 0.00170347309227 0.00860270066635\n",
      "62 15 -0.123056642711 0.00170347309227 0.00860271248364\n",
      "62 20 -0.0636577755213 0.00145343178601 0.00860304953337\n",
      "62 25 -0.0883865505457 0.00145343178601 0.00860347990936\n",
      "62 30 -0.131975755095 0.00145343178601 0.0086039271582\n",
      "62 35 -0.133110016584 0.00145343178601 0.00860420724359\n",
      "62 40 -0.137358367443 0.00145343178601 0.00860447384625\n",
      "62 45 -0.0862679034472 0.00145343178601 0.00860477397596\n",
      "62 50 -0.114854864776 0.00145343178601 0.00860498006516\n",
      "62 55 -0.113048359752 0.00145343178601 0.00860519521222\n",
      "valid_acc 96.83500000000001\n",
      "best valid_acc 96.83500000000001\n",
      "63 0 -0.0909883901477 0.00145343178601 0.00860529551887\n",
      "63 5 -0.102431625128 0.00145343178601 0.00860533269788\n",
      "63 10 -0.107566386461 0.00145343178601 0.0086051693359\n",
      "63 15 -0.085823841393 0.00145343178601 0.00860490083277\n",
      "63 20 -0.117426164448 0.00145343178601 0.00860461610184\n",
      "63 25 -0.111925035715 0.00145343178601 0.00860433824261\n",
      "63 30 -0.0899534970522 0.00145343178601 0.00860406261998\n",
      "63 35 -0.126329541206 0.00145343178601 0.00860366175635\n",
      "63 40 -0.11533677578 0.00145343178601 0.00860341736984\n",
      "63 45 -0.108801595867 0.00145343178601 0.0086033044878\n",
      "63 50 -0.0953629165888 0.00145343178601 0.0086030070256\n",
      "63 55 -0.0953278392553 0.00145343178601 0.00860296349425\n",
      "valid_acc 96.79833333333333\n",
      "64 0 -0.113282188773 0.00145343178601 0.00860329225754\n",
      "64 5 -0.134714320302 0.00145343178601 0.00860406013277\n",
      "64 10 -0.0909047350287 0.00145343178601 0.00860498090802\n",
      "64 15 -0.12355029583 0.00145343178601 0.0086059282784\n",
      "64 20 -0.0813147798181 0.00145343178601 0.00860653344564\n",
      "64 25 -0.0913741141558 0.00145343178601 0.00860711275657\n",
      "64 30 -0.108164571226 0.00145343178601 0.00860772595971\n",
      "64 35 -0.109094783664 0.00145343178601 0.00860852714451\n",
      "64 40 -0.118574522436 0.00145343178601 0.00860953055065\n",
      "64 45 -0.10102725029 0.00145343178601 0.00861066471709\n",
      "64 50 -0.110134929419 0.00145343178601 0.00861185332337\n",
      "64 55 -0.108655758202 0.00145343178601 0.00861317984341\n",
      "valid_acc 96.87166666666667\n",
      "best valid_acc 96.87166666666667\n",
      "65 0 -0.107126124203 0.00145343178601 0.00861438443753\n",
      "65 5 -0.116615585983 0.00145343178601 0.00861540793125\n",
      "65 10 -0.0989694818854 0.00145343178601 0.00861639537406\n",
      "65 15 -0.0914505422115 0.00145343178601 0.00861727402894\n",
      "65 20 -0.111441120505 0.00145343178601 0.00861789843023\n",
      "65 25 -0.111458577216 0.00145343178601 0.00861855001648\n",
      "65 30 -0.114878118038 0.00145343178601 0.00861931708619\n",
      "65 35 -0.0838212892413 0.00145343178601 0.00862023844845\n",
      "65 40 -0.11749843508 0.00145343178601 0.00862109740551\n",
      "65 45 -0.120427526534 0.00145343178601 0.00862190733629\n",
      "65 50 -0.0744878277183 0.00145343178601 0.00862274589514\n",
      "65 55 -0.0782992467284 0.00145343178601 0.00862351550722\n",
      "valid_acc 96.82833333333333\n",
      "66 0 -0.113561466336 0.00145343178601 0.008624319579\n",
      "66 5 -0.106848485768 0.00145343178601 0.00862522825207\n",
      "66 10 -0.115621216595 0.00145343178601 0.00862608176116\n",
      "66 15 -0.112279891968 0.00145343178601 0.00862723422901\n",
      "66 20 -0.117859445512 0.00145343178601 0.00862856728509\n",
      "66 25 -0.0984217971563 0.00145343178601 0.00863006315402\n",
      "66 30 -0.07907345891 0.00145343178601 0.00863169215565\n",
      "66 35 -0.113333813846 0.00145343178601 0.00863333802462\n",
      "66 40 -0.122330330312 0.00145343178601 0.00863493560309\n",
      "66 45 -0.128806352615 0.00145343178601 0.00863681308928\n",
      "66 50 -0.0886457711458 0.00145343178601 0.00863867887513\n",
      "66 55 -0.11393339932 0.00145343178601 0.0086403283031\n",
      "valid_acc 96.88166666666666\n",
      "best valid_acc 96.88166666666666\n",
      "67 0 -0.101435020566 0.00145343178601 0.00864195520738\n",
      "67 5 -0.0776360556483 0.00145343178601 0.00864358329314\n",
      "67 10 -0.109122805297 0.00145343178601 0.00864525587293\n",
      "67 15 -0.106851436198 0.00145343178601 0.00864691411262\n",
      "67 20 -0.100369133055 0.00145343178601 0.00864857205464\n",
      "67 25 -0.0660140737891 0.00145343178601 0.00865023370244\n",
      "67 30 -0.10534632951 0.00145343178601 0.00865184579944\n",
      "67 35 -0.122854493558 0.00145343178601 0.0086532438806\n",
      "67 40 -0.0824962928891 0.00145343178601 0.00865461989453\n",
      "67 45 -0.119402341545 0.00145343178601 0.00865612128672\n",
      "67 50 -0.0779255107045 0.00145343178601 0.00865749372917\n",
      "67 55 -0.104292027652 0.00145343178601 0.00865875843407\n",
      "valid_acc 96.96000000000001\n",
      "best valid_acc 96.96000000000001\n",
      "68 0 -0.0844802036881 0.00145343178601 0.00865983732219\n",
      "68 5 -0.0934960618615 0.00145343178601 0.00866073094115\n",
      "68 10 -0.081580594182 0.00145343178601 0.00866173135296\n",
      "68 15 -0.133490219712 0.00145343178601 0.00866236681622\n",
      "68 20 -0.111139953136 0.00145343178601 0.00866269581037\n",
      "68 25 -0.125029861927 0.00145343178601 0.00866289788056\n",
      "68 30 -0.0773834958673 0.00145343178601 0.00866299826347\n",
      "68 35 -0.148321509361 0.00145343178601 0.00866291027755\n",
      "68 40 -0.10591455549 0.00145343178601 0.00866268240002\n",
      "68 45 -0.098764128983 0.00145343178601 0.00866246125308\n",
      "68 50 -0.0954101011157 0.00145343178601 0.00866220996305\n",
      "68 55 -0.0992642641068 0.00145343178601 0.00866196905238\n",
      "valid_acc 97.02499999999999\n",
      "best valid_acc 97.02499999999999\n",
      "69 0 -0.124886654317 0.00186284865031 0.00866181661241\n",
      "69 5 -0.0731126591563 0.00186284865031 0.00866157477193\n",
      "69 10 -0.0988983586431 0.00186284865031 0.00866136790446\n",
      "69 15 -0.0672662854195 0.00186284865031 0.00866111901664\n",
      "69 20 -0.110367663205 0.00192444170128 0.0086607516541\n",
      "69 25 -0.156543940306 0.00192444170128 0.00866019190409\n",
      "69 30 -0.0799953788519 0.00192444170128 0.00865980809686\n",
      "69 35 -0.0925584882498 0.00192444170128 0.00865959740218\n",
      "69 40 -0.0947591140866 0.00192444170128 0.00865949168191\n",
      "69 45 -0.0908530578017 0.00192444170128 0.00865968271828\n",
      "69 50 -0.111167140305 0.00192444170128 0.00865989099638\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "69 55 -0.117777265608 0.00192444170128 0.00866008561499\n",
      "valid_acc 96.985\n",
      "70 0 -0.146237581968 0.00192444170128 0.00866022638647\n",
      "70 5 -0.0995063707232 0.00192444170128 0.00866047129629\n",
      "70 10 -0.125203356147 0.00192444170128 0.00866062418993\n",
      "70 15 -0.0781003013253 0.00192444170128 0.00866073566851\n",
      "70 20 -0.111474618316 0.00192444170128 0.00866062480446\n",
      "70 25 -0.0831215530634 0.00192444170128 0.00866022369139\n",
      "70 30 -0.0858686640859 0.00192444170128 0.00865987720952\n",
      "70 35 -0.11687541008 0.00192444170128 0.00865948570481\n",
      "70 40 -0.120572656393 0.00192444170128 0.0086591368903\n",
      "70 45 -0.106399551034 0.00192444170128 0.00865894685068\n",
      "70 50 -0.087506107986 0.00192444170128 0.00865895051082\n",
      "70 55 -0.121603310108 0.00192444170128 0.00865869667468\n",
      "valid_acc 97.005\n",
      "71 0 -0.108841709793 0.00192444170128 0.00865844986351\n",
      "71 5 -0.0665321201086 0.00192444170128 0.00865826283142\n",
      "71 10 -0.107123568654 0.00192444170128 0.00865795606699\n",
      "71 15 -0.1049657166 0.00192444170128 0.00865792451026\n",
      "71 20 -0.0894427224994 0.00192444170128 0.00865796420822\n",
      "71 25 -0.109992034733 0.00192444170128 0.00865810015756\n",
      "71 30 -0.1051646173 0.00192444170128 0.00865811114552\n",
      "71 35 -0.119940169156 0.00192444170128 0.00865815607032\n",
      "71 40 -0.0864018127322 0.00192444170128 0.0086581504458\n",
      "71 45 -0.105985812843 0.00192444170128 0.00865827271754\n",
      "71 50 -0.0974984616041 0.00192444170128 0.00865837631892\n",
      "71 55 -0.137864097953 0.00192444170128 0.00865848869662\n",
      "valid_acc 97.00666666666666\n",
      "72 0 -0.0825362429023 0.00192444170128 0.00865864992046\n",
      "72 5 -0.0823930203915 0.00192444170128 0.00865863722233\n",
      "72 10 -0.0991192534566 0.00192444170128 0.00865871709059\n",
      "72 15 -0.0683366507292 0.00192444170128 0.00865898611204\n",
      "72 20 -0.087006315589 0.00192444170128 0.00865926487766\n",
      "72 25 -0.0958141610026 0.00192444170128 0.00865951514373\n",
      "72 30 -0.095176152885 0.00192444170128 0.00865988840179\n",
      "72 35 -0.127299979329 0.00192444170128 0.00866007423954\n",
      "72 40 -0.0955430492759 0.00192444170128 0.00866038278071\n",
      "72 45 -0.108963817358 0.00192444170128 0.00866073835294\n",
      "72 50 -0.094290137291 0.00192444170128 0.00866087389017\n",
      "72 55 -0.103328190744 0.00192444170128 0.00866081564946\n",
      "valid_acc 97.06833333333333\n",
      "best valid_acc 97.06833333333333\n",
      "73 0 -0.0734470784664 0.00192444170128 0.00866083871527\n",
      "73 5 -0.12206929177 0.00192444170128 0.00866081017192\n",
      "73 10 -0.0796822980046 0.00192444170128 0.00866085742941\n",
      "73 15 -0.107343651354 0.00192444170128 0.00866095815061\n",
      "73 20 -0.108632281423 0.00192444170128 0.00866099781489\n",
      "73 25 -0.0951718613505 0.00192444170128 0.00866104143274\n",
      "73 30 -0.11403156817 0.00192444170128 0.00866115624128\n",
      "73 35 -0.134187802672 0.00192444170128 0.00866119899078\n",
      "73 40 -0.0796868726611 0.0016072786753 0.00866114660658\n",
      "73 45 -0.114661984146 0.0016072786753 0.00866098884175\n",
      "73 50 -0.0823412686586 0.0016072786753 0.00866081908555\n",
      "73 55 -0.0807926133275 0.0016072786753 0.00866049961913\n",
      "valid_acc 97.14\n",
      "best valid_acc 97.14\n",
      "74 0 -0.112243652344 0.0016072786753 0.00866005399614\n",
      "74 5 -0.111380770802 0.0016072786753 0.00865945693777\n",
      "74 10 -0.09369892627 0.0016072786753 0.00865882266644\n",
      "74 15 -0.111769720912 0.0016072786753 0.00865825406077\n",
      "74 20 -0.0734055042267 0.0016072786753 0.00865759048713\n",
      "74 25 -0.0779442414641 0.0016072786753 0.008656851118\n",
      "74 30 -0.119884468615 0.0016072786753 0.00865643349228\n",
      "74 35 -0.0860165208578 0.0016072786753 0.00865623329236\n",
      "74 40 -0.10731125623 0.0016072786753 0.00865595714152\n",
      "74 45 -0.0808317735791 0.0016072786753 0.00865564060613\n",
      "74 50 -0.0829745605588 0.0016072786753 0.00865529208599\n",
      "74 55 -0.0949825197458 0.0016072786753 0.00865504771093\n",
      "valid_acc 97.15\n",
      "best valid_acc 97.15\n",
      "75 0 -0.0879603922367 0.0016072786753 0.00865474904551\n",
      "75 5 -0.0794991254807 0.0016072786753 0.0086544876115\n",
      "75 10 -0.0792306885123 0.0016072786753 0.00865419939066\n",
      "75 15 -0.076149828732 0.0016072786753 0.00865404086411\n",
      "75 20 -0.0865246132016 0.0016072786753 0.00865418627951\n",
      "75 25 -0.0782623514533 0.0016072786753 0.00865423296851\n",
      "75 30 -0.0957341343164 0.0016072786753 0.00865421137575\n",
      "75 35 -0.0930489748716 0.0016072786753 0.00865420929224\n",
      "75 40 -0.0959761440754 0.0016072786753 0.00865427163075\n",
      "75 45 -0.0907540172338 0.0016072786753 0.00865424505248\n",
      "75 50 -0.109532527626 0.0016072786753 0.00865409181912\n",
      "75 55 -0.102385520935 0.0016072786753 0.00865384912427\n",
      "valid_acc 97.195\n",
      "best valid_acc 97.195\n",
      "76 0 -0.110606178641 0.0016072786753 0.00865354117493\n",
      "76 5 -0.0808102265 0.0016072786753 0.00865329991621\n",
      "76 10 -0.113218761981 0.0016072786753 0.00865315100879\n",
      "76 15 -0.0972114950418 0.0016072786753 0.00865311066875\n",
      "76 20 -0.0833076760173 0.0016072786753 0.00865320000918\n",
      "76 25 -0.0927546471357 0.0016072786753 0.00865346264637\n",
      "76 30 -0.103200085461 0.0016072786753 0.00865392755111\n",
      "76 35 -0.0870694667101 0.0016072786753 0.00865464296008\n",
      "76 40 -0.109121434391 0.0016072786753 0.00865534189027\n",
      "76 45 -0.102062508464 0.0016072786753 0.00865581181121\n",
      "76 50 -0.116307221353 0.0016072786753 0.00865595759161\n",
      "76 55 -0.0946240946651 0.0016072786753 0.00865649765373\n",
      "valid_acc 97.185\n",
      "77 0 -0.0885739922523 0.0016072786753 0.00865697311278\n",
      "77 5 -0.0877587646246 0.0016072786753 0.00865725323959\n",
      "77 10 -0.105172030628 0.0016072786753 0.0086576002058\n",
      "77 15 -0.0618394091725 0.0016072786753 0.00865793563323\n",
      "77 20 -0.100113131106 0.0016072786753 0.00865839805651\n",
      "77 25 -0.117262266576 0.0016072786753 0.00865904977066\n",
      "77 30 -0.0936843082309 0.0016072786753 0.00865965281513\n",
      "77 35 -0.111952468753 0.0016072786753 0.00866011766255\n",
      "77 40 -0.112788796425 0.0016072786753 0.00866063440377\n",
      "77 45 -0.0904747396708 0.0016072786753 0.00866114029909\n",
      "77 50 -0.134062737226 0.0016072786753 0.00866155842623\n",
      "77 55 -0.0943704247475 0.0016072786753 0.00866171517141\n",
      "valid_acc 97.13000000000001\n",
      "78 0 -0.0942356437445 0.0016072786753 0.00866175552017\n",
      "78 5 -0.0825245901942 0.0016072786753 0.00866175223039\n",
      "78 10 -0.0964596644044 0.0016072786753 0.00866181204783\n",
      "78 15 -0.0746821090579 0.0016072786753 0.00866175838863\n",
      "78 20 -0.117719344795 0.0016072786753 0.00866173853446\n",
      "78 25 -0.0799331218004 0.0016072786753 0.00866162406602\n",
      "78 30 -0.0802394896746 0.0016072786753 0.00866158913103\n",
      "78 35 -0.0976835042238 0.0016072786753 0.00866159566721\n",
      "78 40 -0.0839913189411 0.0016072786753 0.00866169518983\n",
      "78 45 -0.123209394515 0.0016072786753 0.00866175988344\n",
      "78 50 -0.0953496992588 0.0016072786753 0.00866179109949\n",
      "78 55 -0.111219108105 0.0016072786753 0.00866160889839\n",
      "valid_acc 97.17833333333333\n",
      "79 0 -0.0715690106153 0.0016072786753 0.00866113155896\n",
      "79 5 -0.109775573015 0.0016072786753 0.00866060429572\n",
      "79 10 -0.0817470401525 0.0016072786753 0.00866015091847\n",
      "79 15 -0.095172226429 0.0016072786753 0.00865986787658\n",
      "79 20 -0.0962322801352 0.0016072786753 0.00865953746296\n",
      "79 25 -0.10272077471 0.0016072786753 0.00865906979881\n",
      "79 30 -0.0877678915858 0.0016072786753 0.00865862291145\n",
      "79 35 -0.108891949058 0.0016072786753 0.00865795439964\n",
      "79 40 -0.0653351545334 0.0016072786753 0.00865743736517\n",
      "79 45 -0.0935916900635 0.0016072786753 0.00865706479909\n",
      "79 50 -0.0691040977836 0.0016072786753 0.00865674930374\n",
      "79 55 -0.0974672883749 0.0016072786753 0.0086565287378\n",
      "valid_acc 97.295\n",
      "best valid_acc 97.295\n",
      "80 0 -0.07630790025 0.0016072786753 0.00865621907399\n",
      "80 5 -0.0689089596272 0.0016072786753 0.00865564980522\n",
      "80 10 -0.0779112502933 0.0016072786753 0.00865511883717\n",
      "80 15 -0.10534209013 0.0016072786753 0.00865474440704\n",
      "80 20 -0.103810794652 0.0016072786753 0.00865463335818\n",
      "80 25 -0.0741373896599 0.0016072786753 0.00865463810353\n",
      "80 30 -0.0635823830962 0.0016072786753 0.00865453370351\n",
      "80 35 -0.111705869436 0.0016072786753 0.00865441045101\n",
      "80 40 -0.0985020771623 0.0016072786753 0.00865416106923\n",
      "80 45 -0.111971646547 0.0016072786753 0.00865397457554\n",
      "80 50 -0.077499113977 0.0016072786753 0.00865374673603\n",
      "80 55 -0.101287037134 0.0016072786753 0.00865323887758\n",
      "valid_acc 97.27\n",
      "81 0 -0.0841414928436 0.0016072786753 0.00865276520529\n",
      "81 5 -0.0817908868194 0.0016072786753 0.0086522879228\n",
      "81 10 -0.0639883503318 0.0016072786753 0.00865176312191\n",
      "81 15 -0.094598159194 0.0016072786753 0.00865114833789\n",
      "81 20 -0.0862707272172 0.0016072786753 0.0086505832137\n",
      "81 25 -0.0785469934344 0.00144392304637 0.00864993380404\n",
      "81 30 -0.0835560113192 0.00144392304637 0.00864917741356\n",
      "81 35 -0.0840648040175 0.00144392304637 0.00864846650578\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "81 40 -0.0975609049201 0.00144392304637 0.00864782138902\n",
      "81 45 -0.124054789543 0.00144392304637 0.00864721804031\n",
      "81 50 -0.100183710456 0.00144392304637 0.00864664288441\n",
      "81 55 -0.0938979610801 0.00144392304637 0.00864620159057\n",
      "valid_acc 97.35166666666667\n",
      "best valid_acc 97.35166666666667\n",
      "82 0 -0.128769427538 0.00144392304637 0.00864579609801\n",
      "82 5 -0.0941738411784 0.00144392304637 0.00864533163735\n",
      "82 10 -0.0851093307137 0.00144392304637 0.00864483043799\n",
      "82 15 -0.0426878705621 0.00141894062215 0.00864421604013\n",
      "82 20 -0.0864185020328 0.00141894062215 0.00864385224631\n",
      "82 25 -0.0820931270719 0.00141894062215 0.00864364251199\n",
      "82 30 -0.0877967700362 0.00141894062215 0.00864317594056\n",
      "82 35 -0.0958112031221 0.00141894062215 0.00864249946295\n",
      "82 40 -0.0717868730426 0.00141894062215 0.00864156756212\n",
      "82 45 -0.0819352343678 0.00141894062215 0.00864067895052\n",
      "82 50 -0.0806405991316 0.00141894062215 0.0086398368166\n",
      "82 55 -0.0708463862538 0.00141894062215 0.00863896035778\n",
      "valid_acc 97.35333333333334\n",
      "best valid_acc 97.35333333333334\n",
      "83 0 -0.112367630005 0.00141894062215 0.00863806622572\n",
      "83 5 -0.0851169452071 0.00141894062215 0.00863722974533\n",
      "83 10 -0.0892368257046 0.00141894062215 0.00863648277137\n",
      "83 15 -0.112020976841 0.00141894062215 0.00863581964295\n",
      "83 20 -0.0551319420338 0.00141894062215 0.00863505672812\n",
      "83 25 -0.0758791640401 0.00141894062215 0.00863437589144\n",
      "83 30 -0.0707285478711 0.00141894062215 0.00863371114488\n",
      "83 35 -0.0922004133463 0.00141894062215 0.00863302578672\n",
      "83 40 -0.0850088149309 0.00141894062215 0.00863258356901\n",
      "83 45 -0.0746434703469 0.00141894062215 0.0086318472876\n",
      "83 50 -0.080425158143 0.00141894062215 0.00863108110682\n",
      "83 55 -0.0915459543467 0.00141894062215 0.00863039879419\n",
      "valid_acc 97.425\n",
      "best valid_acc 97.425\n",
      "84 0 -0.0737254247069 0.00141894062215 0.0086298455393\n",
      "84 5 -0.0820526704192 0.00141894062215 0.00862937793641\n",
      "84 10 -0.0887342914939 0.00141894062215 0.00862919607866\n",
      "84 15 -0.0927096903324 0.00141894062215 0.00862927634805\n",
      "84 20 -0.0859115123749 0.00141894062215 0.00862927160172\n",
      "84 25 -0.109338440001 0.00141894062215 0.00862920512293\n",
      "84 30 -0.101027853787 0.00141894062215 0.00862929856501\n",
      "84 35 -0.0691998302937 0.00141894062215 0.0086295335493\n",
      "84 40 -0.06516520679 0.00141894062215 0.00862986703719\n",
      "84 45 -0.0532244034111 0.00141894062215 0.00863032936092\n",
      "84 50 -0.0806423649192 0.00141894062215 0.00863091096642\n",
      "84 55 -0.0966326147318 0.00141894062215 0.00863172512037\n",
      "valid_acc 97.40666666666667\n",
      "85 0 -0.0697063952684 0.00141894062215 0.00863231501489\n",
      "85 5 -0.0728194266558 0.00141894062215 0.00863279135234\n",
      "85 10 -0.0836164355278 0.00141894062215 0.00863336917571\n",
      "85 15 -0.086534999311 0.00141894062215 0.00863371997505\n",
      "85 20 -0.114920981228 0.00141894062215 0.00863391515419\n",
      "85 25 -0.0809833407402 0.00141894062215 0.00863386891624\n",
      "85 30 -0.0718056708574 0.00141894062215 0.00863366687039\n",
      "85 35 -0.0894912034273 0.00141894062215 0.00863333045648\n",
      "85 40 -0.0925082415342 0.00141894062215 0.00863307380345\n",
      "85 45 -0.0839214846492 0.00141894062215 0.0086327196174\n",
      "85 50 -0.0994258448482 0.00141894062215 0.00863218467328\n",
      "85 55 -0.0813412740827 0.00141894062215 0.00863159324971\n",
      "valid_acc 97.365\n",
      "86 0 -0.0747392326593 0.00141894062215 0.00863075664798\n",
      "86 5 -0.0783841907978 0.00141894062215 0.00862999187428\n",
      "86 10 -0.079507380724 0.00141894062215 0.00862928071789\n",
      "86 15 -0.077816054225 0.00141894062215 0.00862849782097\n",
      "86 20 -0.0924426913261 0.00141894062215 0.00862760139407\n",
      "86 25 -0.103171430528 0.00141894062215 0.00862680639098\n",
      "86 30 -0.0613676533103 0.00141894062215 0.00862593515766\n",
      "86 35 -0.0972478315234 0.00141894062215 0.0086249227332\n",
      "86 40 -0.0816482901573 0.00141894062215 0.00862401576248\n",
      "86 45 -0.0890037342906 0.00141894062215 0.00862316496156\n",
      "86 50 -0.0831674709916 0.00141894062215 0.00862232894253\n",
      "86 55 -0.0993165224791 0.00141894062215 0.008621354328\n",
      "valid_acc 97.32333333333332\n",
      "87 0 -0.0885712578893 0.00141894062215 0.00862041650707\n",
      "87 5 -0.0826995372772 0.00141894062215 0.0086196728772\n",
      "87 10 -0.0817194357514 0.00141894062215 0.00861890769639\n",
      "87 15 -0.103966653347 0.00141894062215 0.00861818371066\n",
      "87 20 -0.0712078288198 0.00141894062215 0.00861740308285\n",
      "87 25 -0.0829260125756 0.00141894062215 0.00861639196575\n",
      "87 30 -0.116814948618 0.00141894062215 0.00861555467946\n",
      "87 35 -0.0572971999645 0.00141894062215 0.00861476589038\n",
      "87 40 -0.0553003773093 0.00141894062215 0.00861385908683\n",
      "87 45 -0.101054549217 0.00141894062215 0.00861302706223\n",
      "87 50 -0.101160913706 0.00141894062215 0.00861245177209\n",
      "87 55 -0.0809008777142 0.00141894062215 0.00861191017914\n",
      "valid_acc 97.40666666666667\n",
      "88 0 -0.0713933482766 0.00141894062215 0.0086112548664\n",
      "88 5 -0.0564203299582 0.00141894062215 0.0086104495941\n",
      "88 10 -0.0682326629758 0.00141894062215 0.00860981049351\n",
      "88 15 -0.0945727825165 0.00141894062215 0.00860924034995\n",
      "88 20 -0.101082533598 0.00141894062215 0.00860877985502\n",
      "88 25 -0.104846164584 0.00141894062215 0.00860813682745\n",
      "88 30 -0.122074060142 0.00141894062215 0.00860760231159\n",
      "88 35 -0.0984972789884 0.00141894062215 0.00860724267656\n",
      "88 40 -0.0629994720221 0.00141894062215 0.00860668601138\n",
      "88 45 -0.0937047377229 0.00141894062215 0.00860633774378\n",
      "88 50 -0.139725968242 0.00141894062215 0.00860642529845\n",
      "88 55 -0.06166004017 0.00141894062215 0.00860665381495\n",
      "valid_acc 97.33666666666667\n",
      "89 0 -0.0980606228113 0.00141894062215 0.00860686475736\n",
      "89 5 -0.103416182101 0.00141894062215 0.00860717993738\n",
      "89 10 -0.0525201037526 0.00141894062215 0.00860724489021\n",
      "89 15 -0.067940056324 0.00141894062215 0.00860692222061\n",
      "89 20 -0.0914587303996 0.00141894062215 0.00860656279278\n",
      "89 25 -0.0870003551245 0.00141894062215 0.00860621539879\n",
      "89 30 -0.0824097841978 0.00141894062215 0.00860582002424\n",
      "89 35 -0.0798813253641 0.00141894062215 0.00860557984433\n",
      "89 40 -0.0740968063474 0.00141894062215 0.00860535876506\n",
      "89 45 -0.0708170086145 0.00141894062215 0.00860515878857\n",
      "89 50 -0.0906384214759 0.00141894062215 0.00860503856917\n",
      "89 55 -0.0707673057914 0.00141894062215 0.00860491260353\n",
      "valid_acc 97.31833333333333\n",
      "90 0 -0.0828132480383 0.00141894062215 0.00860502173825\n",
      "90 5 -0.0849230661988 0.00141894062215 0.008605159124\n",
      "90 10 -0.0784680768847 0.00141894062215 0.00860521535171\n",
      "90 15 -0.072727009654 0.00141894062215 0.00860542956484\n",
      "90 20 -0.0896737501025 0.00141894062215 0.00860591387494\n",
      "90 25 -0.0808921530843 0.00141894062215 0.0086065428905\n",
      "90 30 -0.0620367713273 0.00141894062215 0.00860702518282\n",
      "90 35 -0.0757176578045 0.00141894062215 0.00860754526383\n",
      "90 40 -0.0935432985425 0.00141894062215 0.00860815861349\n",
      "90 45 -0.0835919901729 0.00141894062215 0.00860868821262\n",
      "90 50 -0.0697886496782 0.00141894062215 0.00860901242727\n",
      "90 55 -0.0835783481598 0.00141894062215 0.00860918985583\n",
      "valid_acc 97.37666666666667\n",
      "91 0 -0.0619025193155 0.00141894062215 0.00860922711492\n",
      "91 5 -0.102156743407 0.00141894062215 0.00860912732847\n",
      "91 10 -0.121036574244 0.00141894062215 0.00860908237866\n",
      "91 15 -0.0693943500519 0.00141894062215 0.00860892250929\n",
      "91 20 -0.0714698061347 0.00141894062215 0.00860868614823\n",
      "91 25 -0.0666559785604 0.00141894062215 0.00860848497001\n",
      "91 30 -0.0925464630127 0.00141894062215 0.00860832018848\n",
      "91 35 -0.10036149621 0.00141894062215 0.00860813339657\n",
      "91 40 -0.096268415451 0.00141894062215 0.00860778137106\n",
      "91 45 -0.096113152802 0.00141894062215 0.00860730435211\n",
      "91 50 -0.117828063667 0.00141894062215 0.00860664494621\n",
      "91 55 -0.0723319724202 0.00141894062215 0.00860615488405\n",
      "valid_acc 97.38166666666666\n",
      "92 0 -0.0931510999799 0.00141894062215 0.00860578179879\n",
      "92 5 -0.105701036751 0.00141894062215 0.00860542227896\n",
      "92 10 -0.0810218900442 0.00141894062215 0.00860515468284\n",
      "92 15 -0.108642891049 0.00141894062215 0.0086050209579\n",
      "92 20 -0.0826241225004 0.00141894062215 0.00860494020707\n",
      "92 25 -0.0610669776797 0.00141894062215 0.00860485174584\n",
      "92 30 -0.0802015513182 0.00141894062215 0.00860476558551\n",
      "92 35 -0.0705508217216 0.00141894062215 0.00860466607195\n",
      "92 40 -0.102399408817 0.00141894062215 0.00860441133681\n",
      "92 45 -0.0933840572834 0.00141894062215 0.00860406744821\n",
      "92 50 -0.0774889960885 0.00141894062215 0.00860359345352\n",
      "92 55 -0.114124931395 0.00141894062215 0.00860296423448\n",
      "valid_acc 97.46666666666667\n",
      "best valid_acc 97.46666666666667\n",
      "93 0 -0.0797840952873 0.00141894062215 0.00860234686956\n",
      "93 5 -0.0771028622985 0.00141894062215 0.00860200942454\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "93 10 -0.0990362018347 0.00141894062215 0.00860155012266\n",
      "93 15 -0.100657187402 0.00141894062215 0.00860112699553\n",
      "93 20 -0.0792678445578 0.00141894062215 0.00860057889277\n",
      "93 25 -0.105209566653 0.00141894062215 0.00859983597546\n",
      "93 30 -0.0902780666947 0.00141894062215 0.00859905258692\n",
      "93 35 -0.0823850482702 0.00141894062215 0.00859838778384\n",
      "93 40 -0.105171583593 0.00141894062215 0.00859773030638\n",
      "93 45 -0.0707477852702 0.00141894062215 0.00859726373399\n",
      "93 50 -0.0678246170282 0.00141894062215 0.0085969110914\n",
      "93 55 -0.102433308959 0.00141894062215 0.00859658688668\n",
      "valid_acc 97.37\n",
      "94 0 -0.0883784145117 0.00141894062215 0.0085962187049\n",
      "94 5 -0.0905069634318 0.00141894062215 0.00859567956817\n",
      "94 10 -0.0915076881647 0.00141894062215 0.00859526389147\n",
      "94 15 -0.0582531280816 0.00141894062215 0.00859495096997\n",
      "94 20 -0.107585050166 0.00141894062215 0.00859469361184\n",
      "94 25 -0.113183021545 0.00141894062215 0.00859445764522\n",
      "94 30 -0.0956960767508 0.00141894062215 0.00859398196372\n",
      "94 35 -0.10699890554 0.00141894062215 0.008593526909\n",
      "94 40 -0.103216223419 0.00141894062215 0.00859325495105\n",
      "94 45 -0.0874372050166 0.00141894062215 0.00859316064794\n",
      "94 50 -0.0916056036949 0.00141894062215 0.0085931068554\n",
      "94 55 -0.076003074646 0.00141894062215 0.00859299319322\n",
      "valid_acc 97.47833333333334\n",
      "best valid_acc 97.47833333333334\n",
      "95 0 -0.0820822641253 0.00141894062215 0.00859291402058\n",
      "95 5 -0.0754160061479 0.00141894062215 0.00859289285643\n",
      "95 10 -0.0838917270303 0.00141894062215 0.00859279672676\n",
      "95 15 -0.0943471863866 0.00141894062215 0.00859290795622\n",
      "95 20 -0.0946198403835 0.00141894062215 0.00859312183815\n",
      "95 25 -0.0912099331617 0.00141894062215 0.00859345773714\n",
      "95 30 -0.106521353126 0.00141894062215 0.00859368091836\n",
      "95 35 -0.0585155561566 0.00141894062215 0.00859390505007\n",
      "95 40 -0.0558866336942 0.00141894062215 0.00859409077585\n",
      "95 45 -0.10624165833 0.00141894062215 0.00859440978183\n",
      "95 50 -0.111678466201 0.00123729478634 0.00859482790576\n",
      "95 55 -0.0833591893315 0.00123729478634 0.00859514510725\n",
      "valid_acc 97.39666666666666\n",
      "96 0 -0.0735331103206 0.00123729478634 0.00859527612806\n",
      "96 5 -0.101878985763 0.00123729478634 0.00859506511718\n",
      "96 10 -0.0794423669577 0.00123729478634 0.00859455659996\n",
      "96 15 -0.0779520794749 0.00123729478634 0.00859384560852\n",
      "96 20 -0.118302889168 0.00123729478634 0.00859300620678\n",
      "96 25 -0.0839014276862 0.00123729478634 0.00859220632987\n",
      "96 30 -0.100522004068 0.00123729478634 0.00859133229528\n",
      "96 35 -0.078311599791 0.00123729478634 0.0085902323632\n",
      "96 40 -0.0563618279994 0.00123729478634 0.00858931965268\n",
      "96 45 -0.0970614477992 0.00123729478634 0.00858866554392\n",
      "96 50 -0.116797782481 0.00123729478634 0.00858825826351\n",
      "96 55 -0.0638025105 0.00123729478634 0.00858800645293\n",
      "valid_acc 97.41499999999999\n",
      "97 0 -0.101203240454 0.00123729478634 0.00858785894453\n",
      "97 5 -0.0693845301867 0.00123729478634 0.00858779901349\n",
      "97 10 -0.0955277308822 0.00123729478634 0.00858785496499\n",
      "97 15 -0.098295584321 0.00123729478634 0.00858782234047\n",
      "97 20 -0.0860266014934 0.00123729478634 0.00858762251975\n",
      "97 25 -0.0708129480481 0.00123729478634 0.00858738843059\n",
      "97 30 -0.0662103369832 0.00123729478634 0.00858711461931\n",
      "97 35 -0.0757665708661 0.00123729478634 0.00858677584235\n",
      "97 40 -0.0454616434872 0.00123729478634 0.0085864061978\n",
      "97 45 -0.0883716121316 0.00123729478634 0.00858603785525\n",
      "97 50 -0.0944436863065 0.00123729478634 0.00858571505303\n",
      "97 55 -0.0943314358592 0.00123729478634 0.00858519930599\n",
      "valid_acc 97.41166666666666\n",
      "98 0 -0.0664731934667 0.00123729478634 0.00858472848773\n",
      "98 5 -0.0828449875116 0.00123729478634 0.00858408661835\n",
      "98 10 -0.0882607847452 0.00123729478634 0.00858344960692\n",
      "98 15 -0.112352617085 0.00123729478634 0.00858292045331\n",
      "98 20 -0.0923969820142 0.00123729478634 0.00858263707249\n",
      "98 25 -0.0881937593222 0.00123729478634 0.00858246490273\n",
      "98 30 -0.0933351069689 0.00123729478634 0.00858224120044\n",
      "98 35 -0.056304205209 0.00123729478634 0.0085818329325\n",
      "98 40 -0.0737554505467 0.00123729478634 0.00858135517064\n",
      "98 45 -0.0719233751297 0.00123729478634 0.00858089438538\n",
      "98 50 -0.0848421901464 0.00123729478634 0.0085805609403\n",
      "98 55 -0.0975360870361 0.00123729478634 0.00858029065231\n",
      "valid_acc 97.41333333333333\n",
      "99 0 -0.0531803704798 0.00123729478634 0.00857993263015\n",
      "99 5 -0.135470956564 0.00123729478634 0.00857957525871\n",
      "99 10 -0.0959521383047 0.00123729478634 0.00857880708422\n",
      "99 15 -0.0776363611221 0.00123729478634 0.00857783242613\n",
      "99 20 -0.0962149724364 0.00123729478634 0.00857667720669\n",
      "99 25 -0.0711016580462 0.00123729478634 0.00857547720997\n",
      "99 30 -0.0939568132162 0.00123729478634 0.00857420793374\n",
      "99 35 -0.0826659351587 0.00123729478634 0.00857297599245\n",
      "99 40 -0.108973406255 0.00123729478634 0.00857189549926\n",
      "99 45 -0.0925891324878 0.00123729478634 0.00857087806012\n",
      "99 50 -0.104162313044 0.00123729478634 0.00857007407093\n",
      "99 55 -0.0820416882634 0.00123729478634 0.00856947025867\n",
      "valid_acc 97.42333333333333\n",
      "100 0 -0.0750000178814 0.00123729478634 0.0085689226458\n",
      "100 5 -0.0657309293747 0.00123729478634 0.00856822919546\n",
      "100 10 -0.0596286542714 0.00123729478634 0.00856754545601\n",
      "100 15 -0.0646235197783 0.00123729478634 0.0085670359735\n",
      "100 20 -0.0872582867742 0.00123729478634 0.00856637268342\n",
      "100 25 -0.0927731320262 0.00123729478634 0.0085655455135\n",
      "100 30 -0.0442542806268 0.00123729478634 0.00856480059879\n",
      "100 35 -0.0746789723635 0.00123729478634 0.00856414715669\n",
      "100 40 -0.0852978006005 0.00123729478634 0.00856338679944\n",
      "100 45 -0.0881797522306 0.00123729478634 0.00856254548953\n",
      "100 50 -0.0950302556157 0.00123729478634 0.00856175137444\n",
      "100 55 -0.0977571234107 0.00123729478634 0.00856087874659\n",
      "valid_acc 97.48333333333333\n",
      "best valid_acc 97.48333333333333\n",
      "101 0 -0.108428962529 0.00123729478634 0.00855984407672\n",
      "101 5 -0.114122264087 0.00123729478634 0.00855861264912\n",
      "101 10 -0.0634317174554 0.00123729478634 0.00855738860144\n",
      "101 15 -0.0638496726751 0.00123729478634 0.00855626707657\n",
      "101 20 -0.079931050539 0.00123729478634 0.00855527873962\n",
      "101 25 -0.070095255971 0.00123729478634 0.00855428643747\n",
      "101 30 -0.0751071050763 0.00123729478634 0.00855329026599\n",
      "101 35 -0.0849717259407 0.00123729478634 0.00855223383714\n",
      "101 40 -0.0897427648306 0.00123729478634 0.00855114989623\n",
      "101 45 -0.0741285085678 0.00123729478634 0.00854984151528\n",
      "101 50 -0.0720004588366 0.00123729478634 0.00854872857301\n",
      "101 55 -0.0905472785234 0.00123729478634 0.0085477296671\n",
      "valid_acc 97.50833333333333\n",
      "best valid_acc 97.50833333333333\n",
      "102 0 -0.089845739305 0.00123729478634 0.00854656517601\n",
      "102 5 -0.0716447681189 0.00123729478634 0.00854553696109\n",
      "102 10 -0.0909218937159 0.00123729478634 0.00854456645698\n",
      "102 15 -0.0809666216373 0.00123729478634 0.00854364766895\n",
      "102 20 -0.0812363997102 0.00123729478634 0.00854280054277\n",
      "102 25 -0.0819268375635 0.00123729478634 0.00854203176653\n",
      "102 30 -0.0568498671055 0.00123729478634 0.00854144154082\n",
      "102 35 -0.0960973873734 0.00123729478634 0.00854068340065\n",
      "102 40 -0.0788075774908 0.00123729478634 0.00854003955017\n",
      "102 45 -0.0680343210697 0.00123729478634 0.00853949434616\n",
      "102 50 -0.0893972888589 0.00123729478634 0.00853880684404\n",
      "102 55 -0.0814416483045 0.00123729478634 0.0085382157267\n",
      "valid_acc 97.46000000000001\n",
      "103 0 -0.0705569162965 0.00123729478634 0.00853769691036\n",
      "103 5 -0.0758556723595 0.00123729478634 0.00853729720067\n",
      "103 10 -0.0900779739022 0.00123729478634 0.00853693072182\n",
      "103 15 -0.0884407162666 0.00123729478634 0.00853657031213\n",
      "103 20 -0.0712249949574 0.00123729478634 0.00853638631276\n",
      "103 25 -0.0730844289064 0.00123729478634 0.00853593599989\n",
      "103 30 -0.0784025341272 0.00123729478634 0.0085355759244\n",
      "103 35 -0.0872563570738 0.00123729478634 0.00853515640208\n",
      "103 40 -0.0850855335593 0.00123729478634 0.00853447281398\n",
      "103 45 -0.070868588984 0.00123729478634 0.00853379463145\n",
      "103 50 -0.077318765223 0.00123729478634 0.00853316377674\n",
      "103 55 -0.0768366828561 0.00123729478634 0.0085324328483\n",
      "valid_acc 97.49166666666666\n",
      "104 0 -0.100034698844 0.00123729478634 0.0085314392818\n",
      "104 5 -0.0694353431463 0.00123729478634 0.00853032397772\n",
      "104 10 -0.0667195618153 0.00123729478634 0.00852922996433\n",
      "104 15 -0.0951741486788 0.00123729478634 0.0085282603607\n",
      "104 20 -0.0609192587435 0.00123729478634 0.00852750060893\n",
      "104 25 -0.0790335088968 0.00123729478634 0.00852675292504\n",
      "104 30 -0.0821447297931 0.00123729478634 0.00852618890605\n",
      "104 35 -0.0843654498458 0.00123729478634 0.00852569712339\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "104 40 -0.0616552196443 0.00123729478634 0.00852519028833\n",
      "104 45 -0.0741441622376 0.00123729478634 0.00852465521464\n",
      "104 50 -0.0648273900151 0.00123729478634 0.00852418266759\n",
      "104 55 -0.0958342254162 0.00123729478634 0.00852398834967\n",
      "valid_acc 97.49166666666666\n",
      "105 0 -0.0704851821065 0.00123729478634 0.00852369439717\n",
      "105 5 -0.076631359756 0.00123729478634 0.00852329458317\n",
      "105 10 -0.0607818439603 0.00123729478634 0.0085231697135\n",
      "105 15 -0.0663150250912 0.00123729478634 0.00852335341391\n",
      "105 20 -0.078627705574 0.00123729478634 0.00852366544364\n",
      "105 25 -0.0718285962939 0.00123729478634 0.00852387292871\n",
      "105 30 -0.062826320529 0.00123729478634 0.00852397709678\n",
      "105 35 -0.0550065897405 0.00123729478634 0.00852414519302\n",
      "105 40 -0.0878590643406 0.00123729478634 0.00852410933793\n",
      "105 45 -0.101986847818 0.00123729478634 0.00852393827437\n",
      "105 50 -0.0786976367235 0.00123729478634 0.00852396562444\n",
      "105 55 -0.0698708817363 0.00123729478634 0.00852389439259\n",
      "valid_acc 97.485\n",
      "106 0 -0.0598132014275 0.00123729478634 0.00852363949962\n",
      "106 5 -0.0582835748792 0.00123729478634 0.00852333079933\n",
      "106 10 -0.0804223567247 0.00123729478634 0.00852288450921\n",
      "106 15 -0.0929196998477 0.00123729478634 0.00852237319176\n",
      "106 20 -0.0444799512625 0.00123729478634 0.00852192401899\n",
      "106 25 -0.0957295671105 0.00123729478634 0.0085214863985\n",
      "106 30 -0.10347186029 0.00123729478634 0.00852123630827\n",
      "106 35 -0.0975277200341 0.00123729478634 0.00852090153863\n",
      "106 40 -0.0754678174853 0.00123729478634 0.00852057230164\n",
      "106 45 -0.0891851186752 0.00123729478634 0.00852039203803\n",
      "106 50 -0.0956585481763 0.00123729478634 0.00852014823642\n",
      "106 55 -0.0652710199356 0.00123729478634 0.00851988755764\n",
      "valid_acc 97.50166666666667\n",
      "107 0 -0.0556911341846 0.00123729478634 0.0085197097719\n",
      "107 5 -0.0705890133977 0.00123729478634 0.00851941088181\n",
      "107 10 -0.0783482864499 0.00123729478634 0.00851893711623\n",
      "107 15 -0.0729248896241 0.00123729478634 0.00851831894612\n",
      "107 20 -0.0660282373428 0.00123729478634 0.00851773738019\n",
      "107 25 -0.0933035686612 0.00123729478634 0.00851709188025\n",
      "107 30 -0.083516843617 0.00123729478634 0.00851633813977\n",
      "107 35 -0.111822172999 0.00123729478634 0.00851553516854\n",
      "107 40 -0.116722032428 0.00123729478634 0.00851502309034\n",
      "107 45 -0.0612749420106 0.00123729478634 0.00851448895331\n",
      "107 50 -0.0543158352375 0.00123729478634 0.0085139490877\n",
      "107 55 -0.0739836543798 0.00123729478634 0.00851350064388\n",
      "valid_acc 97.52166666666666\n",
      "best valid_acc 97.52166666666666\n",
      "108 0 -0.0685018599033 0.00123729478634 0.00851302336519\n",
      "108 5 -0.0793786868453 0.00123729478634 0.00851245776565\n",
      "108 10 -0.11523861438 0.00123729478634 0.00851183949557\n",
      "108 15 -0.0870579257607 0.00123729478634 0.00851122428548\n",
      "108 20 -0.0579432547092 0.00123729478634 0.0085107730975\n",
      "108 25 -0.0734962373972 0.00123729478634 0.00851017037129\n",
      "108 30 -0.0586109571159 0.00123729478634 0.0085096529244\n",
      "108 35 -0.0687739998102 0.00123729478634 0.00850916334004\n",
      "108 40 -0.0782635882497 0.00123729478634 0.00850891759867\n",
      "108 45 -0.0775130316615 0.00123729478634 0.00850882781945\n",
      "108 50 -0.0957521647215 0.00123729478634 0.00850893270652\n",
      "108 55 -0.107951998711 0.00123729478634 0.00850893308524\n",
      "valid_acc 97.57666666666667\n",
      "best valid_acc 97.57666666666667\n",
      "109 0 -0.072912491858 0.00123729478634 0.00850871109409\n",
      "109 5 -0.0577322952449 0.00123729478634 0.00850865697793\n",
      "109 10 -0.0879808440804 0.00123729478634 0.00850873781515\n",
      "109 15 -0.0814574509859 0.00123729478634 0.0085090335861\n",
      "109 20 -0.110687315464 0.00123729478634 0.00850925215736\n",
      "109 25 -0.0787896215916 0.00123729478634 0.00850949842697\n",
      "109 30 -0.109718747437 0.00123729478634 0.00850967803461\n",
      "109 35 -0.0925499647856 0.00123729478634 0.00850971450496\n",
      "109 40 -0.0750905126333 0.00123729478634 0.00850979522086\n",
      "109 45 -0.114564493299 0.00123729478634 0.00850972270423\n",
      "109 50 -0.0890782848001 0.00123729478634 0.00850950090756\n",
      "109 55 -0.0834459885955 0.00123729478634 0.00850931587121\n",
      "valid_acc 97.60833333333333\n",
      "best valid_acc 97.60833333333333\n",
      "110 0 -0.0675135999918 0.00123729478634 0.00850923135783\n",
      "110 5 -0.0818265005946 0.00123729478634 0.00850925658022\n",
      "110 10 -0.0870934575796 0.00123729478634 0.00850929905054\n",
      "110 15 -0.0639109835029 0.00123729478634 0.00850909571079\n",
      "110 20 -0.0539832226932 0.00123729478634 0.0085088497151\n",
      "110 25 -0.0735206007957 0.00123729478634 0.00850848771852\n",
      "110 30 -0.0934466794133 0.00123729478634 0.00850809019781\n",
      "110 35 -0.0677864179015 0.00123729478634 0.00850755500425\n",
      "110 40 -0.0549255944788 0.00123729478634 0.00850692889972\n",
      "110 45 -0.112841643393 0.00123729478634 0.00850648562319\n",
      "110 50 -0.0669947117567 0.00123729478634 0.00850612289487\n",
      "110 55 -0.0975746661425 0.00123729478634 0.00850570844459\n",
      "valid_acc 97.62166666666666\n",
      "best valid_acc 97.62166666666666\n",
      "111 0 -0.067581795156 0.00123729478634 0.00850514538148\n",
      "111 5 -0.0916974022985 0.00123729478634 0.00850426664443\n",
      "111 10 -0.100771635771 0.00123729478634 0.00850352299194\n",
      "111 15 -0.0680275708437 0.00123729478634 0.00850287822349\n",
      "111 20 -0.0791470780969 0.00123729478634 0.00850240513158\n",
      "111 25 -0.0840922743082 0.00123729478634 0.00850199480598\n",
      "111 30 -0.0735763087869 0.00123729478634 0.00850163705026\n",
      "111 35 -0.0820247605443 0.00123729478634 0.00850107163027\n",
      "111 40 -0.0828725248575 0.00123729478634 0.00850046147014\n",
      "111 45 -0.0989707410336 0.00123729478634 0.00850003085741\n",
      "111 50 -0.0613635405898 0.00123729478634 0.00849967902127\n",
      "111 55 -0.0853556990623 0.00123729478634 0.00849922159692\n",
      "valid_acc 97.69666666666666\n",
      "best valid_acc 97.69666666666666\n",
      "112 0 -0.0750535055995 0.00123729478634 0.00849873724242\n",
      "112 5 -0.0779263973236 0.00123729478634 0.00849837195411\n",
      "112 10 -0.0873284414411 0.00123729478634 0.00849780743093\n",
      "112 15 -0.0502610616386 0.00123729478634 0.00849710488624\n",
      "112 20 -0.0866624563932 0.00123729478634 0.00849636458698\n",
      "112 25 -0.0617248415947 0.00123729478634 0.00849544146025\n",
      "112 30 -0.0859745293856 0.00123729478634 0.00849454786735\n",
      "112 35 -0.0754656046629 0.00123729478634 0.00849354292639\n",
      "112 40 -0.081566169858 0.00123729478634 0.00849245717499\n",
      "112 45 -0.0785251483321 0.00123729478634 0.00849143135826\n",
      "112 50 -0.076964892447 0.00123729478634 0.00849031416108\n",
      "112 55 -0.0888579115272 0.00123729478634 0.00848930113784\n",
      "valid_acc 97.64\n",
      "113 0 -0.077436208725 0.00123729478634 0.00848822014815\n",
      "113 5 -0.0617379248142 0.00123729478634 0.0084870806458\n",
      "113 10 -0.115488126874 0.00123729478634 0.00848572289449\n",
      "113 15 -0.0856933966279 0.00123729478634 0.00848434763769\n",
      "113 20 -0.0648255944252 0.00123729478634 0.00848302682105\n",
      "113 25 -0.0766652598977 0.00123729478634 0.008481688084\n",
      "113 30 -0.0677208080888 0.00123729478634 0.00848044200954\n",
      "113 35 -0.0807998105884 0.00123729478634 0.00847924934275\n",
      "113 40 -0.102952338755 0.00123729478634 0.00847820447466\n",
      "113 45 -0.0880364477634 0.00123729478634 0.00847687228406\n",
      "113 50 -0.0635592862964 0.00123729478634 0.00847544100087\n",
      "113 55 -0.0557297989726 0.00123729478634 0.00847406103076\n",
      "valid_acc 97.655\n",
      "114 0 -0.0546215958893 0.00123729478634 0.00847286195844\n",
      "114 5 -0.10862582922 0.00123729478634 0.00847171529163\n",
      "114 10 -0.065229691565 0.00123729478634 0.00847062683247\n",
      "114 15 -0.111818768084 0.00123729478634 0.00846951609389\n",
      "114 20 -0.0825883448124 0.00123729478634 0.00846832355557\n",
      "114 25 -0.0624726563692 0.00123729478634 0.00846724925859\n",
      "114 30 -0.0671531483531 0.00123729478634 0.00846604638923\n",
      "114 35 -0.0693243443966 0.00123729478634 0.00846503738886\n",
      "114 40 -0.0841180160642 0.00123729478634 0.00846406314152\n",
      "114 45 -0.0503720119596 0.00123729478634 0.00846311573004\n",
      "114 50 -0.0657198354602 0.00123729478634 0.00846213712451\n",
      "114 55 -0.0637596622109 0.00123729478634 0.00846120701824\n",
      "valid_acc 97.68833333333333\n",
      "115 0 -0.084738522768 0.00123729478634 0.00846039386275\n",
      "115 5 -0.0779355242848 0.00123729478634 0.00845970871414\n",
      "115 10 -0.0726752355695 0.00123729478634 0.00845870394458\n",
      "115 15 -0.0616068802774 0.00123729478634 0.00845760955634\n",
      "115 20 -0.0945881009102 0.00123729478634 0.00845637860066\n",
      "115 25 -0.0622935332358 0.00123729478634 0.00845514489055\n",
      "115 30 -0.110780380666 0.00123729478634 0.00845395003673\n",
      "115 35 -0.0549970008433 0.00123729478634 0.00845282717866\n",
      "115 40 -0.0901931300759 0.00123729478634 0.00845177200743\n",
      "115 45 -0.0571649000049 0.00123729478634 0.00845068964522\n",
      "115 50 -0.0696914792061 0.00123729478634 0.00844984375885\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "115 55 -0.0775852799416 0.00123729478634 0.00844912895918\n",
      "valid_acc 97.74000000000001\n",
      "best valid_acc 97.74000000000001\n",
      "116 0 -0.0596221908927 0.00123729478634 0.00844841505381\n",
      "116 5 -0.107194967568 0.00123729478634 0.00844783667368\n",
      "116 10 -0.0652734264731 0.00123729478634 0.00844710367597\n",
      "116 15 -0.0815025195479 0.00123729478634 0.00844657317679\n",
      "116 20 -0.0988908857107 0.00123729478634 0.00844601445676\n",
      "116 25 -0.0816899985075 0.00123729478634 0.00844541993758\n",
      "116 30 -0.0673439651728 0.00123729478634 0.00844485893309\n",
      "116 35 -0.0938362777233 0.00123729478634 0.00844445666398\n",
      "116 40 -0.0682886019349 0.00123729478634 0.00844413189266\n",
      "116 45 -0.0635283440351 0.00123729478634 0.00844388434556\n",
      "116 50 -0.0546054244041 0.00123729478634 0.00844381144927\n",
      "116 55 -0.0618528053164 0.00123729478634 0.00844381178075\n",
      "valid_acc 97.7\n",
      "117 0 -0.0671283900738 0.00123729478634 0.00844367811008\n",
      "117 5 -0.0931346267462 0.00123729478634 0.00844325570791\n",
      "117 10 -0.060307379812 0.00123729478634 0.00844300743959\n",
      "117 15 -0.0535619966686 0.00123729478634 0.00844282974763\n",
      "117 20 -0.0634506866336 0.00123729478634 0.00844263228999\n",
      "117 25 -0.0475828014314 0.00123729478634 0.00844249527472\n",
      "117 30 -0.129564166069 0.000762344788621 0.00844224370067\n",
      "117 35 -0.0904180407524 0.000762344788621 0.00844183420971\n",
      "117 40 -0.047210816294 0.000762344788621 0.00844140763774\n",
      "117 45 -0.0699418187141 0.000762344788621 0.00844121067534\n",
      "117 50 -0.0747048035264 0.000762344788621 0.00844085474349\n",
      "117 55 -0.0776805579662 0.000762344788621 0.00844048531531\n",
      "valid_acc 97.67666666666666\n",
      "118 0 -0.0866101756692 0.000762344788621 0.00844025173218\n",
      "118 5 -0.0702663436532 0.000762344788621 0.00843994832775\n",
      "118 10 -0.0733698010445 0.000762344788621 0.00843957343955\n",
      "118 15 -0.0753593817353 0.000762344788621 0.00843925146613\n",
      "118 20 -0.0834337994456 0.000762344788621 0.00843901911499\n",
      "118 25 -0.0685508027673 0.000762344788621 0.00843900104784\n",
      "118 30 -0.0543609671295 0.000762344788621 0.00843904129073\n",
      "118 35 -0.090659506619 0.000762344788621 0.00843912698639\n",
      "118 40 -0.0680155977607 0.000762344788621 0.00843920711042\n",
      "118 45 -0.0584086216986 0.000762344788621 0.00843937296921\n",
      "118 50 -0.0594251789153 0.000762344788621 0.00843938553958\n",
      "118 55 -0.07430190593 0.000762344788621 0.00843931153644\n",
      "valid_acc 97.72333333333333\n",
      "119 0 -0.0563872642815 0.000762344788621 0.00843939713868\n",
      "119 5 -0.0956832841039 0.000762344788621 0.00843938839499\n",
      "119 10 -0.0773753300309 0.000762344788621 0.00843945694974\n",
      "119 15 -0.0827277377248 0.000762344788621 0.00843946813396\n",
      "119 20 -0.0471230521798 0.000762344788621 0.00843965988464\n",
      "119 25 -0.0528950877488 0.000762344788621 0.00843996275021\n",
      "119 30 -0.0676106512547 0.000762344788621 0.00843998740222\n",
      "119 35 -0.0714897438884 0.000762344788621 0.00844009456554\n",
      "119 40 -0.0650040134788 0.000762344788621 0.00844026187142\n",
      "119 45 -0.0850830674171 0.000762344788621 0.00844045819757\n",
      "119 50 -0.0731522142887 0.000762344788621 0.00844066910533\n",
      "119 55 -0.0848053693771 0.000762344788621 0.00844082927851\n",
      "valid_acc 97.69\n",
      "120 0 -0.0665325298905 0.000762344788621 0.00844082703525\n",
      "120 5 -0.0796574801207 0.000762344788621 0.00844094016711\n",
      "120 10 -0.0681387484074 0.000762344788621 0.00844101096175\n",
      "120 15 -0.10137937963 0.000762344788621 0.00844094081117\n",
      "120 20 -0.0580896958709 0.000762344788621 0.00844074109679\n",
      "120 25 -0.0790998786688 0.000762344788621 0.00844042842885\n",
      "120 30 -0.0833328962326 0.000762344788621 0.00843980507104\n",
      "120 35 -0.0897103026509 0.000762344788621 0.0084392154417\n",
      "120 40 -0.0744957998395 0.000762344788621 0.00843847724329\n",
      "120 45 -0.0448882989585 0.000762344788621 0.00843767532189\n",
      "120 50 -0.0852330178022 0.000762344788621 0.00843693355602\n",
      "120 55 -0.100174061954 0.000762344788621 0.00843615870847\n",
      "valid_acc 97.74833333333333\n",
      "best valid_acc 97.74833333333333\n",
      "121 0 -0.0783262178302 0.000762344788621 0.00843538634193\n",
      "121 5 -0.105916880071 0.000762344788621 0.00843455509166\n",
      "121 10 -0.0726233124733 0.000762344788621 0.00843366846539\n",
      "121 15 -0.105492912233 0.000762344788621 0.00843269803517\n",
      "121 20 -0.0674384683371 0.000762344788621 0.00843178203258\n",
      "121 25 -0.0701675489545 0.000762344788621 0.00843082101384\n",
      "121 30 -0.0715042427182 0.000762344788621 0.00842998057268\n",
      "121 35 -0.0575961135328 0.000762344788621 0.00842926461302\n",
      "121 40 -0.0956624150276 0.000762344788621 0.0084288081088\n",
      "121 45 -0.0848351940513 0.000762344788621 0.00842839875788\n",
      "121 50 -0.0547146089375 0.000762344788621 0.00842784802049\n",
      "121 55 -0.0709104761481 0.000762344788621 0.00842725495418\n",
      "valid_acc 97.76\n",
      "best valid_acc 97.76\n",
      "122 0 -0.0604766346514 0.000762344788621 0.0084266195309\n",
      "122 5 -0.0606000348926 0.000762344788621 0.00842584280997\n",
      "122 10 -0.109874509275 0.000762344788621 0.00842502261307\n",
      "122 15 -0.0652386769652 0.000762344788621 0.00842414523095\n",
      "122 20 -0.0974064618349 0.000762344788621 0.00842294750615\n",
      "122 25 -0.0443544052541 0.000762344788621 0.00842152862928\n",
      "122 30 -0.0866802483797 0.000762344788621 0.00842006552355\n",
      "122 35 -0.0738915279508 0.000762344788621 0.00841852637184\n",
      "122 40 -0.0730509608984 0.000762344788621 0.00841702383671\n",
      "122 45 -0.0929746553302 0.000762344788621 0.00841597658949\n",
      "122 50 -0.0807375684381 0.000762344788621 0.00841485221386\n",
      "122 55 -0.084650285542 0.000762344788621 0.00841329125683\n",
      "valid_acc 97.71166666666666\n",
      "123 0 -0.0653633773327 0.000762344788621 0.00841169887148\n",
      "123 5 -0.0695368349552 0.000762344788621 0.00841007811577\n",
      "123 10 -0.081052325666 0.000762344788621 0.00840828740881\n",
      "123 15 -0.0609515830874 0.000762344788621 0.00840652750485\n",
      "123 20 -0.0643498897552 0.000762344788621 0.00840486856424\n",
      "123 25 -0.106960244477 0.000762344788621 0.0084032136812\n",
      "123 30 -0.0777352452278 0.000762344788621 0.00840161103854\n",
      "123 35 -0.0619962476194 0.000762344788621 0.00840028581566\n",
      "123 40 -0.0647146180272 0.000762344788621 0.00839885811648\n",
      "123 45 -0.0560713708401 0.000762344788621 0.00839736596873\n",
      "123 50 -0.0902571231127 0.000762344788621 0.00839585266713\n",
      "123 55 -0.0730942189693 0.000762344788621 0.00839417593772\n",
      "valid_acc 97.70333333333333\n",
      "124 0 -0.0497078374028 0.000762344788621 0.0083925382239\n",
      "124 5 -0.0738144889474 0.000762344788621 0.00839090090203\n",
      "124 10 -0.0553535073996 0.000762344788621 0.00838907414795\n",
      "124 15 -0.0669016689062 0.000762344788621 0.00838716993208\n",
      "124 20 -0.0611715465784 0.000762344788621 0.00838534056843\n",
      "124 25 -0.053399913013 0.000762344788621 0.00838356426765\n",
      "124 30 -0.101405754685 0.000762344788621 0.00838203490794\n",
      "124 35 -0.0887322574854 0.000762344788621 0.00838047809962\n",
      "124 40 -0.0958536192775 0.000762344788621 0.00837903743373\n",
      "124 45 -0.0601658113301 0.000762344788621 0.00837731345988\n",
      "124 50 -0.0785955190659 0.000762344788621 0.00837575347594\n",
      "124 55 -0.0566205121577 0.000762344788621 0.00837430523439\n",
      "valid_acc 97.695\n",
      "125 0 -0.0825630798936 0.000762344788621 0.00837268297463\n",
      "125 5 -0.0833193138242 0.000762344788621 0.00837099434477\n",
      "125 10 -0.0408172048628 0.000762344788621 0.00836920445575\n",
      "125 15 -0.0784768536687 0.000762344788621 0.00836727260527\n",
      "125 20 -0.10249530524 0.000762344788621 0.00836535487346\n",
      "125 25 -0.071081340313 0.000762344788621 0.00836347682264\n",
      "125 30 -0.0739743188024 0.000762344788621 0.00836180958337\n",
      "125 35 -0.0712590441108 0.000762344788621 0.00836027478913\n",
      "125 40 -0.0737786442041 0.000762344788621 0.00835891815034\n",
      "125 45 -0.079849742353 0.000762344788621 0.00835758993735\n",
      "125 50 -0.0960313156247 0.000762344788621 0.00835618713497\n",
      "125 55 -0.078784391284 0.000762344788621 0.00835488695268\n",
      "valid_acc 97.705\n",
      "126 0 -0.0540517643094 0.000762344788621 0.00835358621018\n",
      "126 5 -0.0820605605841 0.000762344788621 0.00835202198287\n",
      "126 10 -0.0646690428257 0.000762344788621 0.00835032565815\n",
      "126 15 -0.0761071369052 0.000762344788621 0.00834864978819\n",
      "126 20 -0.0707815140486 0.000762344788621 0.0083468514387\n",
      "126 25 -0.0967537537217 0.000762344788621 0.00834496842517\n",
      "126 30 -0.0665077120066 0.000762344788621 0.0083431748413\n",
      "126 35 -0.0807179287076 0.000762344788621 0.00834144660022\n",
      "126 40 -0.0628166049719 0.000762344788621 0.00833970950961\n",
      "126 45 -0.114791966975 0.000762344788621 0.00833805930342\n",
      "126 50 -0.0544621311128 0.000762344788621 0.0083363167241\n",
      "126 55 -0.0671966671944 0.000762344788621 0.00833433847243\n",
      "valid_acc 97.76666666666667\n",
      "best valid_acc 97.76666666666667\n",
      "127 0 -0.0570539310575 0.000762344788621 0.00833238216391\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "127 5 -0.086433596909 0.000762344788621 0.008330490644\n",
      "127 10 -0.0973401665688 0.000762344788621 0.00832883371564\n",
      "127 15 -0.0621613375843 0.000762344788621 0.00832711659454\n",
      "127 20 -0.0568967163563 0.000762344788621 0.008325554206\n",
      "127 25 -0.0595213025808 0.000762344788621 0.00832401811353\n",
      "127 30 -0.0685626417398 0.000762344788621 0.00832223327114\n",
      "127 35 -0.0618659816682 0.000762344788621 0.00832033238593\n",
      "127 40 -0.0545894987881 0.000762344788621 0.00831856478146\n",
      "127 45 -0.0598545372486 0.000762344788621 0.00831683990938\n",
      "127 50 -0.0539620704949 0.000762344788621 0.00831525073211\n",
      "127 55 -0.0634645298123 0.000762344788621 0.00831375527803\n",
      "valid_acc 97.74000000000001\n",
      "128 0 -0.0779337137938 0.000762344788621 0.00831214737022\n",
      "128 5 -0.0696460232139 0.000762344788621 0.00831062203106\n",
      "128 10 -0.0770926252007 0.000762344788621 0.00830915538609\n",
      "128 15 -0.0554972141981 0.000762344788621 0.00830780666587\n",
      "128 20 -0.0760338380933 0.000762344788621 0.00830644655965\n",
      "128 25 -0.0724214315414 0.000762344788621 0.00830490081275\n",
      "128 30 -0.0626402124763 0.000762344788621 0.00830334892019\n",
      "128 35 -0.0710528343916 0.000762344788621 0.00830174126364\n",
      "128 40 -0.063050724566 0.000762344788621 0.00830028061434\n",
      "128 45 -0.085592828691 0.000762344788621 0.00829892087725\n",
      "128 50 -0.0607318021357 0.000762344788621 0.00829767955466\n",
      "128 55 -0.0639213770628 0.000762344788621 0.0082965419991\n",
      "valid_acc 97.76666666666667\n",
      "best valid_acc 97.76666666666667\n",
      "129 0 -0.0507707409561 0.000762344788621 0.00829558132435\n",
      "129 5 -0.0687344819307 0.000762344788621 0.0082946291159\n",
      "129 10 -0.0530597306788 0.000762344788621 0.00829358810324\n",
      "129 15 -0.0732375234365 0.000762344788621 0.00829245268004\n",
      "129 20 -0.0836585238576 0.000762344788621 0.00829129558187\n",
      "129 25 -0.0524697639048 0.000762344788621 0.00829026981686\n",
      "129 30 -0.0737210437655 0.000762344788621 0.00828939216635\n",
      "129 35 -0.0644117817283 0.000762344788621 0.0082886267629\n",
      "129 40 -0.0435635596514 0.000762344788621 0.00828801424152\n",
      "129 45 -0.0858192816377 0.000762344788621 0.00828732975422\n",
      "129 50 -0.0843476429582 0.000762344788621 0.00828667011906\n",
      "129 55 -0.0871720761061 0.000762344788621 0.00828578346373\n",
      "valid_acc 97.82833333333333\n",
      "best valid_acc 97.82833333333333\n",
      "130 0 -0.0426442660391 0.000762344788621 0.00828471554591\n",
      "130 5 -0.0588208734989 0.000762344788621 0.00828349754139\n",
      "130 10 -0.0972023308277 0.000762344788621 0.00828225668997\n",
      "130 15 -0.074305832386 0.000762344788621 0.00828115932157\n",
      "130 20 -0.0971571877599 0.000762344788621 0.00828015855776\n",
      "130 25 -0.0661771595478 0.000762344788621 0.00827929067539\n",
      "130 30 -0.059233032167 0.000762344788621 0.00827834241876\n",
      "130 35 -0.0800602138042 0.000762344788621 0.00827726575134\n",
      "130 40 -0.0730359479785 0.000762344788621 0.00827626856119\n",
      "130 45 -0.0632472708821 0.000762344788621 0.00827515583315\n",
      "130 50 -0.0486622005701 0.000762344788621 0.00827422067046\n",
      "130 55 -0.0650416314602 0.000762344788621 0.008273480993\n",
      "valid_acc 97.81833333333333\n",
      "131 0 -0.0987810641527 0.000762344788621 0.00827290356988\n",
      "131 5 -0.0828170627356 0.000762344788621 0.00827247616495\n",
      "131 10 -0.0647831857204 0.000762344788621 0.00827201785362\n",
      "131 15 -0.0707319900393 0.000762344788621 0.00827145282924\n",
      "131 20 -0.0811614394188 0.000762344788621 0.00827097787268\n",
      "131 25 -0.0836171880364 0.000762344788621 0.00827062073769\n",
      "131 30 -0.0600072517991 0.000762344788621 0.00827034549069\n",
      "131 35 -0.0824390351772 0.000762344788621 0.00827006625694\n",
      "131 40 -0.0791160389781 0.000762344788621 0.00826992090793\n",
      "131 45 -0.0669732242823 0.000762344788621 0.00826981911384\n",
      "131 50 -0.0764059275389 0.000762344788621 0.0082698652321\n",
      "131 55 -0.0734179764986 0.000762344788621 0.00826993042018\n",
      "valid_acc 97.77\n",
      "132 0 -0.0623615719378 0.000762344788621 0.00827005289677\n",
      "132 5 -0.0484572276473 0.000762344788621 0.00827023850144\n",
      "132 10 -0.066672347486 0.000762344788621 0.00827054813793\n",
      "132 15 -0.0591899566352 0.000762344788621 0.00827097593267\n",
      "132 20 -0.0886401683092 0.000762344788621 0.00827127151383\n",
      "132 25 -0.0606077462435 0.000762344788621 0.00827146240811\n",
      "132 30 -0.093718983233 0.000762344788621 0.0082717111451\n",
      "132 35 -0.0815929248929 0.000762344788621 0.00827185166239\n",
      "132 40 -0.0673601925373 0.000762344788621 0.00827199948465\n",
      "132 45 -0.0734012648463 0.000762344788621 0.00827203536589\n",
      "132 50 -0.0695832744241 0.000762344788621 0.00827206059797\n",
      "132 55 -0.0806691423059 0.000762344788621 0.00827219556822\n",
      "valid_acc 97.7\n",
      "133 0 -0.1009497419 0.000762344788621 0.00827258031438\n",
      "133 5 -0.0773185044527 0.000762344788621 0.00827286226002\n",
      "133 10 -0.0466437228024 0.000762344788621 0.00827290933767\n",
      "133 15 -0.0707076787949 0.000762344788621 0.00827279908478\n",
      "133 20 -0.0711367279291 0.000762344788621 0.00827274932064\n",
      "133 25 -0.0449945442379 0.000762344788621 0.00827241074638\n",
      "133 30 -0.088366612792 0.000762344788621 0.00827201502731\n",
      "133 35 -0.0922241732478 0.000762344788621 0.00827156051231\n",
      "133 40 -0.0774910673499 0.000762344788621 0.00827098991946\n",
      "133 45 -0.0647606626153 0.000762344788621 0.0082704459154\n",
      "133 50 -0.0696142241359 0.000762344788621 0.0082699744791\n",
      "133 55 -0.0732924491167 0.000762344788621 0.00826950056031\n",
      "valid_acc 97.74666666666667\n",
      "134 0 -0.0615707188845 0.000762344788621 0.00826929040962\n",
      "134 5 -0.056371435523 0.000762344788621 0.00826929252519\n",
      "134 10 -0.0717979222536 0.000762344788621 0.00826927262291\n",
      "134 15 -0.074287250638 0.000762344788621 0.00826923390533\n",
      "134 20 -0.0779603943229 0.000762344788621 0.00826910773932\n",
      "134 25 -0.086766153574 0.000762344788621 0.00826896591832\n",
      "134 30 -0.0788016021252 0.000762344788621 0.00826868837149\n",
      "134 35 -0.0755685567856 0.000762344788621 0.00826843716295\n",
      "134 40 -0.0867995023727 0.000762344788621 0.00826825947756\n",
      "134 45 -0.0694412365556 0.000762344788621 0.0082680600281\n",
      "134 50 -0.0588779635727 0.000762344788621 0.00826779952474\n",
      "134 55 -0.081685975194 0.000762344788621 0.00826737131161\n",
      "valid_acc 97.72833333333332\n",
      "135 0 -0.0755457729101 0.000762344788621 0.00826697816826\n",
      "135 5 -0.0849626660347 0.000762344788621 0.00826651442827\n",
      "135 10 -0.0844180285931 0.000762344788621 0.00826604773031\n",
      "135 15 -0.0563862845302 0.000762344788621 0.00826545632318\n",
      "135 20 -0.0580283291638 0.000762344788621 0.00826483432524\n",
      "135 25 -0.0772726088762 0.000762344788621 0.00826423359365\n",
      "135 30 -0.0587227791548 0.000762344788621 0.00826369491104\n",
      "135 35 -0.0649900659919 0.000762344788621 0.00826304663407\n",
      "135 40 -0.0452188029885 0.000762344788621 0.00826247307367\n",
      "135 45 -0.0879812911153 0.000762344788621 0.00826207044696\n",
      "135 50 -0.0827312320471 0.000762344788621 0.00826160195393\n",
      "135 55 -0.0538205653429 0.000762344788621 0.00826122896707\n",
      "valid_acc 97.73166666666665\n",
      "136 0 -0.0578945837915 0.000762344788621 0.00826083932792\n",
      "136 5 -0.0583081804216 0.000762344788621 0.00826050710228\n",
      "136 10 -0.0830132141709 0.000762344788621 0.0082602536688\n",
      "136 15 -0.0776889771223 0.000762344788621 0.00826010879108\n",
      "136 20 -0.0705025494099 0.000762344788621 0.00826001814811\n",
      "136 25 -0.0572974830866 0.000762344788621 0.00826004664647\n",
      "136 30 -0.0843372046947 0.000762344788621 0.00826008250763\n",
      "136 35 -0.075347006321 0.000762344788621 0.00825999441093\n",
      "136 40 -0.0422527454793 0.000762344788621 0.00825977152244\n",
      "136 45 -0.0570513904095 0.000762344788621 0.00825954494945\n",
      "136 50 -0.0452879443765 0.000762344788621 0.00825940317594\n",
      "136 55 -0.0845055505633 0.000762344788621 0.00825938766651\n",
      "valid_acc 97.81333333333333\n",
      "137 0 -0.0488177984953 0.000762344788621 0.00825928178413\n",
      "137 5 -0.0582691691816 0.000762344788621 0.00825910362243\n",
      "137 10 -0.0686258226633 0.000762344788621 0.00825886238351\n",
      "137 15 -0.0990688353777 0.000762344788621 0.00825848382856\n",
      "137 20 -0.0564485304058 0.000762344788621 0.00825803918197\n",
      "137 25 -0.092960819602 0.000762344788621 0.00825758522938\n",
      "137 30 -0.049439843744 0.000762344788621 0.00825699853241\n",
      "137 35 -0.0580725073814 0.000762344788621 0.00825634540438\n",
      "137 40 -0.0503129474819 0.000762344788621 0.00825573420033\n",
      "137 45 -0.06225271523 0.000762344788621 0.00825511509545\n",
      "137 50 -0.0673077777028 0.000762344788621 0.00825457700354\n",
      "137 55 -0.0774489566684 0.000762344788621 0.00825402097688\n",
      "valid_acc 97.82166666666666\n",
      "138 0 -0.069086804986 0.000762344788621 0.00825354240658\n",
      "138 5 -0.0798651203513 0.000762344788621 0.00825314982274\n",
      "138 10 -0.0465828031301 0.000762344788621 0.00825304573035\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "138 15 -0.0709323137999 0.000762344788621 0.0082530284761\n",
      "138 20 -0.0920824334025 0.000762344788621 0.00825297917256\n",
      "138 25 -0.0751685574651 0.000762344788621 0.0082529643448\n",
      "138 30 -0.0998077318072 0.000762344788621 0.00825286781692\n",
      "138 35 -0.0514114126563 0.000762344788621 0.00825285374013\n",
      "138 40 -0.0789420083165 0.000762344788621 0.00825289372671\n",
      "138 45 -0.0615340098739 0.000762344788621 0.00825290380286\n",
      "138 50 -0.0702671036124 0.000762344788621 0.00825303010209\n",
      "138 55 -0.0764487087727 0.000762344788621 0.00825299816823\n",
      "valid_acc 97.82333333333332\n",
      "139 0 -0.0745214670897 0.000762344788621 0.00825282344691\n",
      "139 5 -0.0370079651475 0.000762344788621 0.00825272907708\n",
      "139 10 -0.0702407285571 0.000762344788621 0.00825267403099\n",
      "139 15 -0.0618415474892 0.000762344788621 0.00825295597016\n",
      "139 20 -0.0527727901936 0.000762344788621 0.00825335028572\n",
      "139 25 -0.0946982428432 0.000762344788621 0.00825353439932\n",
      "139 30 -0.0733321532607 0.000762344788621 0.00825367045994\n",
      "139 35 -0.0693167597055 0.000762344788621 0.00825390878734\n",
      "139 40 -0.0621462278068 0.000762344788621 0.00825413012267\n",
      "139 45 -0.0619215928018 0.000762344788621 0.00825428997614\n",
      "139 50 -0.0816403403878 0.000762344788621 0.00825449493796\n",
      "139 55 -0.0968001559377 0.000762344788621 0.00825472884992\n",
      "valid_acc 97.91666666666666\n",
      "best valid_acc 97.91666666666666\n",
      "140 0 -0.0607253536582 0.000762344788621 0.00825529073682\n",
      "140 5 -0.0894156396389 0.000762344788621 0.00825599388054\n",
      "140 10 -0.0462303683162 0.000762344788621 0.00825661036526\n",
      "140 15 -0.0863262265921 0.000762344788621 0.00825706355177\n",
      "140 20 -0.0665619000793 0.000762344788621 0.00825748043973\n",
      "140 25 -0.0676319673657 0.000762344788621 0.00825783632781\n",
      "140 30 -0.0954220294952 0.000762344788621 0.00825807591023\n",
      "140 35 -0.0900722816586 0.000762344788621 0.00825824801984\n",
      "140 40 -0.0611869432032 0.000762344788621 0.00825845024097\n",
      "140 45 -0.0607452392578 0.000762344788621 0.00825863771406\n",
      "140 50 -0.0722457319498 0.000762344788621 0.00825877151331\n",
      "140 55 -0.0624556727707 0.000762344788621 0.0082586602507\n",
      "valid_acc 97.91333333333333\n",
      "141 0 -0.0470170117915 0.000762344788621 0.00825853218413\n",
      "141 5 -0.0632579922676 0.000762344788621 0.00825829791604\n",
      "141 10 -0.0891215726733 0.000762344788621 0.00825796791056\n",
      "141 15 -0.0660468190908 0.000762344788621 0.00825738175614\n",
      "141 20 -0.0440850444138 0.000762344788621 0.00825675615454\n",
      "141 25 -0.0547144189477 0.000762344788621 0.00825626331067\n",
      "141 30 -0.0964588597417 0.000762344788621 0.00825574697743\n",
      "141 35 -0.0742675364017 0.000762344788621 0.00825538510331\n",
      "141 40 -0.113248489797 0.000762344788621 0.00825518798513\n",
      "141 45 -0.0772907435894 0.000762344788621 0.00825517091229\n",
      "141 50 -0.0541375167668 0.000762344788621 0.00825515371468\n",
      "141 55 -0.055519271642 0.000762344788621 0.00825496630128\n",
      "valid_acc 97.89\n",
      "142 0 -0.0708887204528 0.000762344788621 0.00825448933126\n",
      "142 5 -0.08285292238 0.000762344788621 0.00825413252463\n",
      "142 10 -0.0658776089549 0.000762344788621 0.00825385289381\n",
      "142 15 -0.0827668085694 0.000762344788621 0.00825345485012\n",
      "142 20 -0.0779755860567 0.000762344788621 0.00825286016901\n",
      "142 25 -0.0822961851954 0.00101264420024 0.00825227987513\n",
      "142 30 -0.0884418785572 0.00101264420024 0.00825186347706\n",
      "142 35 -0.0593502037227 0.00101264420024 0.0082514918635\n",
      "142 40 -0.0591032020748 0.00101264420024 0.00825104065038\n",
      "142 45 -0.0670926868916 0.00101264420024 0.00825056672695\n",
      "142 50 -0.0735342502594 0.00101264420024 0.00824982606341\n",
      "142 55 -0.0633791685104 0.00101264420024 0.008249293421\n",
      "valid_acc 97.85666666666667\n",
      "143 0 -0.05173715204 0.00101264420024 0.00824881899666\n",
      "143 5 -0.0724009275436 0.00101264420024 0.00824821620231\n",
      "143 10 -0.0853478386998 0.00101264420024 0.0082475194584\n",
      "143 15 -0.0612627603114 0.00101264420024 0.008246626146\n",
      "143 20 -0.060131046921 0.00101264420024 0.008245659424\n",
      "143 25 -0.0775646045804 0.00101264420024 0.00824490223937\n",
      "143 30 -0.0626392513514 0.00101264420024 0.00824430346732\n",
      "143 35 -0.0607229694724 0.00101264420024 0.00824361190204\n",
      "143 40 -0.0479367338121 0.00101264420024 0.0082429834461\n",
      "143 45 -0.0827127471566 0.00101264420024 0.00824225858571\n",
      "143 50 -0.0656166523695 0.00101264420024 0.00824174963188\n",
      "143 55 -0.0830916687846 0.00101264420024 0.00824118337268\n",
      "valid_acc 97.82\n",
      "144 0 -0.0383965298533 0.00101264420024 0.00824043492925\n",
      "144 5 -0.0566506832838 0.00101264420024 0.00823961898906\n",
      "144 10 -0.0544312186539 0.00101264420024 0.0082389651413\n",
      "144 15 -0.0752392113209 0.00101264420024 0.00823846073293\n",
      "144 20 -0.0572374425828 0.00101264420024 0.00823811253306\n",
      "144 25 -0.0526541918516 0.00101264420024 0.008237800719\n",
      "144 30 -0.0511471889913 0.00101264420024 0.00823741830131\n",
      "144 35 -0.0538096353412 0.00101264420024 0.00823693187129\n",
      "144 40 -0.0719830915332 0.00101264420024 0.00823641604229\n",
      "144 45 -0.089624799788 0.00101264420024 0.00823594050695\n",
      "144 50 -0.0791763812304 0.00101264420024 0.00823540714905\n",
      "144 55 -0.0616283901036 0.00101264420024 0.00823466475739\n",
      "valid_acc 97.79666666666667\n",
      "145 0 -0.0771232992411 0.00101264420024 0.00823378393609\n",
      "145 5 -0.100415714085 0.00101264420024 0.00823280369244\n",
      "145 10 -0.0633392855525 0.00101264420024 0.00823174112237\n",
      "145 15 -0.0731217712164 0.00101264420024 0.00823060585026\n",
      "145 20 -0.0822182297707 0.00101264420024 0.00822954310361\n",
      "145 25 -0.0834996253252 0.00101264420024 0.00822855217663\n",
      "145 30 -0.0700069218874 0.00101264420024 0.00822746264663\n",
      "145 35 -0.0780630260706 0.00101264420024 0.00822642972722\n",
      "145 40 -0.0660915821791 0.00101264420024 0.00822548086626\n",
      "145 45 -0.056288536638 0.00101264420024 0.00822503499552\n",
      "145 50 -0.0510251261294 0.00101264420024 0.00822446652169\n",
      "145 55 -0.0576292015612 0.00101264420024 0.0082237868885\n",
      "valid_acc 97.82666666666667\n",
      "146 0 -0.0754484757781 0.000516628049579 0.00822299375838\n",
      "146 5 -0.0547017902136 0.000516628049579 0.00822229886369\n",
      "146 10 -0.0505649223924 0.000516628049579 0.00822152957047\n",
      "146 15 -0.064820304513 0.000516628049579 0.00822070003907\n",
      "146 20 -0.0524137914181 0.000516628049579 0.00821981802854\n",
      "146 25 -0.0676836818457 0.000516628049579 0.00821903902823\n",
      "146 30 -0.0918631106615 0.000516628049579 0.00821838604978\n",
      "146 35 -0.0736009329557 0.000516628049579 0.00821788986366\n",
      "146 40 -0.0683404803276 0.000516628049579 0.00821719785137\n",
      "146 45 -0.0678222924471 0.000516628049579 0.00821641940954\n",
      "146 50 -0.04849993065 0.000516628049579 0.00821559379399\n",
      "146 55 -0.0849573612213 0.000516628049579 0.00821494543659\n",
      "valid_acc 97.81833333333333\n",
      "147 0 -0.059888664633 0.000516628049579 0.0082144806347\n",
      "147 5 -0.112543806434 0.000516628049579 0.00821421530143\n",
      "147 10 -0.0662379562855 0.000516628049579 0.00821413878157\n",
      "147 15 -0.0738143026829 0.000516628049579 0.00821421436183\n",
      "147 20 -0.0447892472148 0.000516628049579 0.00821418584404\n",
      "147 25 -0.0853550955653 0.000516628049579 0.0082141434152\n",
      "147 30 -0.0785209983587 0.000516628049579 0.00821385934276\n",
      "147 35 -0.0867305770516 0.000516628049579 0.00821345895243\n",
      "147 40 -0.0972736850381 0.000516628049579 0.00821316203035\n",
      "147 45 -0.124520160258 0.000516628049579 0.00821270248573\n",
      "147 50 -0.0774681270123 0.000516628049579 0.00821215690647\n",
      "147 55 -0.0602855756879 0.000516628049579 0.00821170771316\n",
      "valid_acc 97.83833333333334\n",
      "148 0 -0.0482968203723 0.000516628049579 0.00821141488884\n",
      "148 5 -0.0833096280694 0.000516628049579 0.00821112959972\n",
      "148 10 -0.0709754154086 0.000516628049579 0.0082108580419\n",
      "148 15 -0.0591361634433 0.000516628049579 0.00821041986409\n",
      "148 20 -0.0504680089653 0.000516628049579 0.00820974197246\n",
      "148 25 -0.0722075626254 0.000516628049579 0.00820904773793\n",
      "148 30 -0.0547730959952 0.000516628049579 0.00820848911347\n",
      "148 35 -0.0654357820749 0.000516628049579 0.00820785201897\n",
      "148 40 -0.0796099677682 0.000516628049579 0.00820703968937\n",
      "148 45 -0.0721339806914 0.000516628049579 0.00820606094628\n",
      "148 50 -0.044470589608 0.000516628049579 0.00820516144253\n",
      "148 55 -0.0575586631894 0.000516628049579 0.00820433017386\n",
      "valid_acc 97.86333333333333\n",
      "149 0 -0.0768102034926 0.000516628049579 0.00820341988236\n",
      "149 5 -0.0765541568398 0.000324490472957 0.00820279384967\n",
      "149 10 -0.0736191645265 0.000324490472957 0.00820244396404\n",
      "149 15 -0.0695067495108 0.000324490472957 0.00820248762431\n",
      "149 20 -0.0894365608692 0.000324490472957 0.00820252613595\n",
      "149 25 -0.0589589700103 0.000324490472957 0.00820247959107\n",
      "149 30 -0.0641132891178 0.000324490472957 0.00820234760083\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "149 35 -0.0709662586451 0.000324490472957 0.00820214881914\n",
      "149 40 -0.0792170315981 0.000324490472957 0.00820180559863\n",
      "149 45 -0.0809343904257 0.000324490472957 0.00820138794892\n",
      "149 50 -0.0500726476312 0.000324490472957 0.00820099459149\n",
      "149 55 -0.076802380383 0.000324490472957 0.00820066614878\n",
      "valid_acc 97.86166666666666\n",
      "150 0 -0.0627814978361 0.000324490472957 0.00820035320325\n",
      "150 5 -0.0819283276796 0.000324490472957 0.00820020362319\n",
      "150 10 -0.0762176439166 0.000324490472957 0.008199967467\n",
      "150 15 -0.0761633589864 0.000324490472957 0.0081997112541\n",
      "150 20 -0.0693892836571 0.000324490472957 0.0081993043735\n",
      "150 25 -0.068565748632 0.000324490472957 0.00819871613193\n",
      "150 30 -0.0720161348581 0.000324490472957 0.00819786045809\n",
      "150 35 -0.0780110657215 0.000324490472957 0.00819717747194\n",
      "150 40 -0.0837850049138 0.000324490472957 0.00819651594046\n",
      "150 45 -0.0735961794853 0.000324490472957 0.00819586098988\n",
      "150 50 -0.0643697530031 0.000324490472957 0.00819518201487\n",
      "150 55 -0.0648746937513 0.000324490472957 0.00819416844611\n",
      "valid_acc 97.925\n",
      "best valid_acc 97.925\n",
      "151 0 -0.058133456856 0.000324490472957 0.00819309158989\n",
      "151 5 -0.0696351602674 0.000324490472957 0.00819205052539\n",
      "151 10 -0.0535520464182 0.000324490472957 0.00819084007853\n",
      "151 15 -0.063379868865 0.000324490472957 0.00818960959166\n",
      "151 20 -0.0654366090894 0.000324490472957 0.00818841535631\n",
      "151 25 -0.0613012984395 0.000324490472957 0.00818723891917\n",
      "151 30 -0.0524962544441 0.000324490472957 0.0081861167253\n",
      "151 35 -0.0535004884005 0.000324490472957 0.00818503758536\n",
      "151 40 -0.0478232242167 0.000324490472957 0.00818385913135\n",
      "151 45 -0.0969044715166 0.000324490472957 0.00818254925591\n",
      "151 50 -0.0523031316698 0.000324490472957 0.00818131591702\n",
      "151 55 -0.0960649698973 0.000324490472957 0.00817992730636\n",
      "valid_acc 97.95\n",
      "best valid_acc 97.95\n",
      "152 0 -0.0929762050509 0.000324490472957 0.00817861599013\n",
      "152 5 -0.08102080971 0.000324490472957 0.00817754546091\n",
      "152 10 -0.0511086136103 0.000324490472957 0.00817630125357\n",
      "152 15 -0.0582754760981 0.000324490472957 0.00817485710248\n",
      "152 20 -0.0514067709446 0.000324490472957 0.00817336996096\n",
      "152 25 -0.0571726299822 0.000324490472957 0.00817211295284\n",
      "152 30 -0.0673996731639 0.000324490472957 0.00817091913451\n",
      "152 35 -0.0519559048116 0.000324490472957 0.00816991058798\n",
      "152 40 -0.0847736820579 0.000324490472957 0.00816903920315\n",
      "152 45 -0.0485632605851 0.000324490472957 0.00816807288961\n",
      "152 50 -0.0800431519747 0.000324490472957 0.00816711728231\n",
      "152 55 -0.0571641996503 0.000324490472957 0.00816608121535\n",
      "valid_acc 97.92666666666666\n",
      "153 0 -0.0800043791533 0.000324490472957 0.00816509760738\n",
      "153 5 -0.0560860298574 0.000324490472957 0.00816396484408\n",
      "153 10 -0.0382642447948 0.000324490472957 0.00816292148034\n",
      "153 15 -0.0529094487429 0.000324490472957 0.00816175791851\n",
      "153 20 -0.0969648808241 0.000324490472957 0.00816058242989\n",
      "153 25 -0.0797471776605 0.000324490472957 0.00815937581153\n",
      "153 30 -0.0832465514541 0.000324490472957 0.00815805025725\n",
      "153 35 -0.0698807314038 0.000324490472957 0.0081565886076\n",
      "153 40 -0.0657498911023 0.000324490472957 0.00815519588307\n",
      "153 45 -0.0776610895991 0.000324490472957 0.00815369783498\n",
      "153 50 -0.0727678388357 0.000324490472957 0.00815200892458\n",
      "153 55 -0.0529845803976 0.000324490472957 0.00815006512307\n",
      "valid_acc 97.96666666666667\n",
      "best valid_acc 97.96666666666667\n",
      "154 0 -0.0431373268366 0.000324490472957 0.00814786659736\n",
      "154 5 -0.112674951553 0.000324490472957 0.00814559367294\n",
      "154 10 -0.0478245355189 0.000324490472957 0.00814335509953\n",
      "154 15 -0.0725145116448 0.000324490472957 0.00814094643913\n",
      "154 20 -0.0499970316887 0.000324490472957 0.00813862420373\n",
      "154 25 -0.0859519317746 0.000324490472957 0.00813646132127\n",
      "154 30 -0.0586583837867 0.000324490472957 0.00813443076467\n",
      "154 35 -0.0625004246831 0.000324490472957 0.00813250419155\n",
      "154 40 -0.0395851805806 0.000324490472957 0.00813078521394\n",
      "154 45 -0.0790430009365 0.000324490472957 0.00812924911257\n",
      "154 50 -0.0757898837328 0.000324490472957 0.00812790440019\n",
      "154 55 -0.0633842274547 0.000324490472957 0.00812670222577\n",
      "valid_acc 97.89666666666666\n",
      "155 0 -0.0746804326773 0.000324490472957 0.00812539823583\n",
      "155 5 -0.0392304062843 0.000324490472957 0.00812377638846\n",
      "155 10 -0.0738813951612 0.000324490472957 0.00812211202747\n",
      "155 15 -0.058252196759 0.000324490472957 0.00812032801807\n",
      "155 20 -0.0639399215579 0.000324490472957 0.00811854757307\n",
      "155 25 -0.0727566480637 0.000324490472957 0.00811678292458\n",
      "155 30 -0.0643769577146 0.000324490472957 0.00811496675859\n",
      "155 35 -0.0996553525329 0.000324490472957 0.0081131814429\n",
      "155 40 -0.0717458873987 0.000324490472957 0.00811116036024\n",
      "155 45 -0.0605088323355 0.000324490472957 0.00810918026683\n",
      "155 50 -0.071341432631 0.000324490472957 0.008107392893\n",
      "155 55 -0.0651364251971 0.000324490472957 0.00810579676161\n",
      "valid_acc 97.94500000000001\n",
      "156 0 -0.0484539866447 0.000324490472957 0.00810434234896\n",
      "156 5 -0.0708776637912 0.000324490472957 0.00810295380739\n",
      "156 10 -0.0983394905925 0.000324490472957 0.00810171181416\n",
      "156 15 -0.0753909349442 0.000324490472957 0.00810079987112\n",
      "156 20 -0.0600528307259 0.000324490472957 0.00809979051497\n",
      "156 25 -0.0642060413957 0.000324490472957 0.0080988082004\n",
      "156 30 -0.0602772794664 0.000324490472957 0.00809800950124\n",
      "156 35 -0.0516914054751 0.000324490472957 0.00809728964239\n",
      "156 40 -0.0781826451421 0.000324490472957 0.00809641365269\n",
      "156 45 -0.0821710973978 0.000324490472957 0.0080954118419\n",
      "156 50 -0.0671810135245 0.000324490472957 0.00809436832261\n",
      "156 55 -0.0495342686772 0.000324490472957 0.00809344570406\n",
      "valid_acc 97.92833333333333\n",
      "157 0 -0.0901565253735 0.000324490472957 0.00809250484861\n",
      "157 5 -0.0929388850927 0.000324490472957 0.00809160539533\n",
      "157 10 -0.0382674410939 0.000324490472957 0.0080908626464\n",
      "157 15 -0.0679403692484 0.000324490472957 0.0080902483186\n",
      "157 20 -0.0676929801702 0.000324490472957 0.00808957960978\n",
      "157 25 -0.0712420344353 0.000324490472957 0.008089055809\n",
      "157 30 -0.0928184837103 0.000324490472957 0.00808876403124\n",
      "157 35 -0.0659141242504 0.000324490472957 0.00808867945648\n",
      "157 40 -0.0628209337592 0.000324490472957 0.00808847435137\n",
      "157 45 -0.0688549652696 0.000324490472957 0.00808825970266\n",
      "157 50 -0.0716508552432 0.000324490472957 0.00808822327177\n",
      "157 55 -0.064849793911 0.000324490472957 0.00808805343529\n",
      "valid_acc 97.91499999999999\n",
      "158 0 -0.0554915219545 0.000324490472957 0.00808781946038\n",
      "158 5 -0.0623013079166 0.000324490472957 0.00808744415938\n",
      "158 10 -0.073855265975 0.000324490472957 0.00808713148869\n",
      "158 15 -0.0872651562095 0.000324490472957 0.00808682465937\n",
      "158 20 -0.0584688186646 0.000324490472957 0.00808664536526\n",
      "158 25 -0.0437897443771 0.000324490472957 0.00808634304608\n",
      "158 30 -0.0638414770365 0.000324490472957 0.008085868051\n",
      "158 35 -0.071875244379 0.000324490472957 0.00808556404294\n",
      "158 40 -0.0606438852847 0.000324490472957 0.00808539932284\n",
      "158 45 -0.0502445809543 0.000324490472957 0.00808511932403\n",
      "158 50 -0.0672451704741 0.000324490472957 0.00808481158863\n",
      "158 55 -0.0803048834205 0.000324490472957 0.00808443230316\n",
      "valid_acc 97.90666666666667\n",
      "159 0 -0.0644901245832 0.000324490472957 0.0080841523398\n",
      "159 5 -0.0623270832002 0.000324490472957 0.00808374688743\n",
      "159 10 -0.0540651232004 0.000324490472957 0.00808331770146\n",
      "159 15 -0.054104745388 0.000324490472957 0.00808307652373\n",
      "159 20 -0.0824093520641 0.000324490472957 0.00808274877902\n",
      "159 25 -0.0542710535228 0.000324490472957 0.00808236473407\n",
      "159 30 -0.0835718736053 0.000324490472957 0.00808192304322\n",
      "159 35 -0.0706605613232 0.000324490472957 0.00808151508619\n",
      "159 40 -0.0596647337079 0.000324490472957 0.00808116754601\n",
      "159 45 -0.0774570703506 0.000324490472957 0.00808089893851\n",
      "159 50 -0.0690695047379 0.000324490472957 0.00808042505499\n",
      "159 55 -0.0530467890203 0.000324490472957 0.00808005899651\n",
      "valid_acc 97.925\n",
      "160 0 -0.0507185794413 0.000324490472957 0.00807953758189\n",
      "160 5 -0.0670196637511 0.000324490472957 0.00807889750858\n",
      "160 10 -0.0624308697879 0.000324490472957 0.00807822386816\n",
      "160 15 -0.0691903159022 0.000324490472957 0.00807772538791\n",
      "160 20 -0.0562195368111 0.000324490472957 0.0080775459799\n",
      "160 25 -0.0500140078366 0.000324490472957 0.00807734117919\n",
      "160 30 -0.0494282133877 0.000324490472957 0.00807711166542\n",
      "160 35 -0.0819079428911 0.000324490472957 0.00807681674033\n",
      "160 40 -0.0668709650636 0.000324490472957 0.00807645989912\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "160 45 -0.0609503313899 0.000324490472957 0.00807595469117\n",
      "160 50 -0.0848718062043 0.000324490472957 0.00807538517323\n",
      "160 55 -0.0835366100073 0.000324490472957 0.00807477896173\n",
      "valid_acc 97.89166666666667\n",
      "161 0 -0.0697878971696 0.000324490472957 0.00807418694961\n",
      "161 5 -0.0570758618414 0.000324490472957 0.00807388465892\n",
      "161 10 -0.0875971391797 0.000324490472957 0.00807376582696\n",
      "161 15 -0.104996673763 0.000324490472957 0.00807371951721\n",
      "161 20 -0.0716222301126 0.000324490472957 0.0080735316035\n",
      "161 25 -0.0565357096493 0.000324490472957 0.00807335345589\n",
      "161 30 -0.0589729845524 0.000324490472957 0.00807326764427\n",
      "161 35 -0.0620184615254 0.000324490472957 0.00807322361593\n",
      "161 40 -0.0737155899405 0.000324490472957 0.00807305100862\n",
      "161 45 -0.0776428803802 0.000324490472957 0.00807280322511\n",
      "161 50 -0.0724394395947 0.000324490472957 0.00807247644391\n",
      "161 55 -0.0817425251007 0.000324490472957 0.00807227184464\n",
      "valid_acc 97.94\n",
      "162 0 -0.0675295963883 0.000324490472957 0.00807227481207\n",
      "162 5 -0.0481130518019 0.000324490472957 0.00807218031656\n",
      "162 10 -0.0613859146833 0.000324490472957 0.00807197413183\n",
      "162 15 -0.0528338253498 0.000324490472957 0.00807200948775\n",
      "162 20 -0.0783391073346 0.000324490472957 0.00807195720261\n",
      "162 25 -0.0700097382069 0.000324490472957 0.00807193017421\n",
      "162 30 -0.0670532360673 0.000324490472957 0.00807177113595\n",
      "162 35 -0.065740890801 0.000324490472957 0.00807161219335\n",
      "162 40 -0.0488138496876 0.000324490472957 0.00807143589664\n",
      "162 45 -0.071560330689 0.000324490472957 0.00807134496252\n",
      "162 50 -0.0466775298119 0.000324490472957 0.00807127302548\n",
      "162 55 -0.0437228865921 0.000324490472957 0.00807097721908\n",
      "valid_acc 97.93833333333333\n",
      "163 0 -0.0412881299853 0.000324490472957 0.00807060080454\n",
      "163 5 -0.0712758526206 0.000324490472957 0.00807008348141\n",
      "163 10 -0.0884583517909 0.000324490472957 0.00806933377904\n",
      "163 15 -0.0807121992111 0.000324490472957 0.0080685516983\n",
      "163 20 -0.0993229746819 0.000324490472957 0.00806772684786\n",
      "163 25 -0.048634853214 0.000324490472957 0.00806691988409\n",
      "163 30 -0.0548220686615 0.000324490472957 0.00806582596339\n",
      "163 35 -0.048568084836 0.000324490472957 0.0080648076307\n",
      "163 40 -0.0727642849088 0.000324490472957 0.00806369973633\n",
      "163 45 -0.0554094612598 0.000324490472957 0.00806241880831\n",
      "163 50 -0.0556350052357 0.000324490472957 0.00806103997688\n",
      "163 55 -0.0757088288665 0.000324490472957 0.00805975875125\n",
      "valid_acc 97.96833333333333\n",
      "best valid_acc 97.96833333333333\n",
      "164 0 -0.0500517562032 0.000324490472957 0.00805849210416\n",
      "164 5 -0.0605274476111 0.000324490472957 0.00805743513353\n",
      "164 10 -0.0615994036198 0.000324490472957 0.00805618507343\n",
      "164 15 -0.0531399548054 0.000324490472957 0.00805479270179\n",
      "164 20 -0.0669540092349 0.000324490472957 0.00805358473654\n",
      "164 25 -0.0465786084533 0.000324490472957 0.00805222825639\n",
      "164 30 -0.0597727149725 0.000324490472957 0.0080508511871\n",
      "164 35 -0.0807549059391 0.000324490472957 0.00804947877345\n",
      "164 40 -0.0575127154589 0.000324490472957 0.00804803253674\n",
      "164 45 -0.0687357187271 0.000324490472957 0.00804640005611\n",
      "164 50 -0.0450219102204 0.000324490472957 0.00804498559168\n",
      "164 55 -0.0596505776048 0.000324490472957 0.00804329425307\n",
      "valid_acc 98.0\n",
      "best valid_acc 98.0\n",
      "165 0 -0.0502949506044 0.000324490472957 0.00804168363241\n",
      "165 5 -0.0466525219381 0.000324490472957 0.0080400753558\n",
      "165 10 -0.0824597999454 0.000324490472957 0.00803833560165\n",
      "165 15 -0.0647506862879 0.000324490472957 0.0080365084957\n",
      "165 20 -0.0880659520626 0.000324490472957 0.00803462676453\n",
      "165 25 -0.0454315878451 0.000324490472957 0.00803281396498\n",
      "165 30 -0.0596486032009 0.000324490472957 0.00803098785579\n",
      "165 35 -0.0325336530805 0.000324490472957 0.00802908034345\n",
      "165 40 -0.0664831399918 0.000324490472957 0.00802726798654\n",
      "165 45 -0.0827516913414 0.000324490472957 0.00802566225579\n",
      "165 50 -0.0857702866197 0.000324490472957 0.00802415070111\n",
      "165 55 -0.0571232661605 0.000324490472957 0.00802258761476\n",
      "valid_acc 97.96166666666667\n",
      "166 0 -0.0625746920705 0.000324490472957 0.00802110253283\n",
      "166 5 -0.062556758523 0.000324490472957 0.00801981993354\n",
      "166 10 -0.0678411573172 0.000324490472957 0.00801858428583\n",
      "166 15 -0.0733964592218 0.000324490472957 0.00801743471094\n",
      "166 20 -0.0865024104714 0.000324490472957 0.00801624789524\n",
      "166 25 -0.0542391054332 0.000324490472957 0.00801522334026\n",
      "166 30 -0.0530106499791 0.000324490472957 0.00801422331665\n",
      "166 35 -0.0708064213395 0.000324490472957 0.00801332234648\n",
      "166 40 -0.0580671988428 0.000324490472957 0.00801229280826\n",
      "166 45 -0.0611880645156 0.000324490472957 0.00801130753964\n",
      "166 50 -0.0908292010427 0.000324490472957 0.00801034487592\n",
      "166 55 -0.107688762248 0.000324490472957 0.00800962309671\n",
      "valid_acc 97.965\n",
      "167 0 -0.0932360067964 0.000324490472957 0.00800898026181\n",
      "167 5 -0.0562986843288 0.000324490472957 0.00800842393681\n",
      "167 10 -0.0599248856306 0.000324490472957 0.00800780399923\n",
      "167 15 -0.050491463393 0.000324490472957 0.00800707019313\n",
      "167 20 -0.0629140362144 0.000324490472957 0.00800624829822\n",
      "167 25 -0.0555473007262 0.000324490472957 0.00800553765064\n",
      "167 30 -0.0690069198608 0.000324490472957 0.00800495603913\n",
      "167 35 -0.056058883667 0.000324490472957 0.00800453752054\n",
      "167 40 -0.0365393534303 0.000324490472957 0.00800401160709\n",
      "167 45 -0.0716867223382 0.000324490472957 0.00800342862248\n",
      "167 50 -0.0662534758449 0.000324490472957 0.00800271276267\n",
      "167 55 -0.0484478846192 0.000324490472957 0.00800198965849\n",
      "valid_acc 98.01166666666667\n",
      "best valid_acc 98.01166666666667\n",
      "168 0 -0.06004710868 0.000324490472957 0.00800140957658\n",
      "168 5 -0.0538910776377 0.000324490472957 0.00800083834282\n",
      "168 10 -0.0717305168509 0.000324490472957 0.00800050624138\n",
      "168 15 -0.0654039680958 0.000324490472957 0.00800026801771\n",
      "168 20 -0.0694623589516 0.000324490472957 0.00800028559591\n",
      "168 25 -0.0822444483638 0.000324490472957 0.0080004062776\n",
      "168 30 -0.0524340644479 0.000324490472957 0.00800064699235\n",
      "168 35 -0.0799081251025 0.000324490472957 0.00800099234638\n",
      "168 40 -0.091461494565 0.000324490472957 0.0080012545782\n",
      "168 45 -0.0519286841154 0.000324490472957 0.00800177932009\n",
      "168 50 -0.056516520679 0.000324490472957 0.00800219552405\n",
      "168 55 -0.0564623177052 0.000324490472957 0.00800243669067\n",
      "valid_acc 98.00166666666667\n",
      "169 0 -0.0655037984252 0.000324490472957 0.00800251341096\n",
      "169 5 -0.0734178498387 0.000324490472957 0.00800257095153\n",
      "169 10 -0.0747204124928 0.000324490472957 0.00800262707396\n",
      "169 15 -0.0441078320146 0.000324490472957 0.00800247728109\n",
      "169 20 -0.0674149170518 0.000324490472957 0.0080024786735\n",
      "169 25 -0.0788682624698 0.000324490472957 0.00800229926561\n",
      "169 30 -0.0604734644294 0.000324490472957 0.00800208289994\n",
      "169 35 -0.0737061724067 0.000324490472957 0.00800191879485\n",
      "169 40 -0.0652908757329 0.000324490472957 0.00800202900307\n",
      "169 45 -0.0558518432081 0.000324490472957 0.00800219518484\n",
      "169 50 -0.0535682290792 0.000324490472957 0.00800253687283\n",
      "169 55 -0.0704638212919 0.000324490472957 0.00800269516658\n",
      "valid_acc 97.94333333333334\n",
      "170 0 -0.0644167810678 0.000324490472957 0.00800279315975\n",
      "170 5 -0.0853357091546 0.000324490472957 0.00800296231232\n",
      "170 10 -0.0628897100687 0.000324490472957 0.00800290408302\n",
      "170 15 -0.0536861047149 0.000324490472957 0.00800280161268\n",
      "170 20 -0.0714630335569 0.000324490472957 0.00800269316752\n",
      "170 25 -0.0554884597659 0.000324490472957 0.00800270586086\n",
      "170 30 -0.0438474230468 0.000324490472957 0.00800272332819\n",
      "170 35 -0.0685933753848 0.000324490472957 0.00800259095388\n",
      "170 40 -0.0628759637475 0.000324490472957 0.00800251691356\n",
      "170 45 -0.0810784175992 0.000324490472957 0.00800249489654\n",
      "170 50 -0.060057554394 0.000324490472957 0.0080023236034\n",
      "170 55 -0.0775748565793 0.000324490472957 0.00800205473908\n",
      "valid_acc 97.99833333333333\n",
      "171 0 -0.0660102665424 0.000324490472957 0.00800174708579\n",
      "171 5 -0.0503076612949 0.000324490472957 0.00800144474852\n",
      "171 10 -0.0430744104087 0.000324490472957 0.0080012204525\n",
      "171 15 -0.0397857017815 0.000324490472957 0.00800120793501\n",
      "171 20 -0.053604837507 0.000324490472957 0.0080012791181\n",
      "171 25 -0.0751259326935 0.000324490472957 0.00800135613771\n",
      "171 30 -0.0624180473387 0.000324490472957 0.00800137836729\n",
      "171 35 -0.0931034013629 0.000324490472957 0.0080012131743\n",
      "171 40 -0.0759162083268 0.000324490472957 0.00800097594374\n",
      "171 45 -0.0752556025982 0.000324490472957 0.00800073272899\n",
      "171 50 -0.0693477392197 0.000324490472957 0.00800050874283\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "171 55 -0.090197853744 0.000324490472957 0.00800030999926\n",
      "valid_acc 97.97833333333334\n",
      "172 0 -0.103900790215 0.000324490472957 0.00800023170461\n",
      "172 5 -0.0449748858809 0.000324490472957 0.00800034903553\n",
      "172 10 -0.0471583120525 0.000324490472957 0.00800053572073\n",
      "172 15 -0.0519707091153 0.000324490472957 0.0080006081617\n",
      "172 20 -0.0715850889683 0.000324490472957 0.00800085926273\n",
      "172 25 -0.0449092648923 0.000324490472957 0.00800131241068\n",
      "172 30 -0.0469944626093 0.000324490472957 0.00800165872834\n",
      "172 35 -0.0587226711214 0.000324490472957 0.00800196386459\n",
      "172 40 -0.0914255976677 0.000324490472957 0.00800206358223\n",
      "172 45 -0.0631159469485 0.000324490472957 0.00800204479354\n",
      "172 50 -0.0622268095613 0.000324490472957 0.00800206939605\n",
      "172 55 -0.0562582314014 0.000324490472957 0.0080019642409\n",
      "valid_acc 97.97333333333333\n",
      "173 0 -0.0494392029941 0.000324490472957 0.00800191442841\n",
      "173 5 -0.0567678138614 0.000324490472957 0.00800182270293\n",
      "173 10 -0.0451760552824 0.000324490472957 0.00800185188128\n",
      "173 15 -0.0847744643688 0.000324490472957 0.00800202534312\n",
      "173 20 -0.0381918996572 0.000324490472957 0.00800239924997\n",
      "173 25 -0.0351329296827 0.000324490472957 0.00800301126576\n",
      "173 30 -0.0635759234428 0.000324490472957 0.0080034700621\n",
      "173 35 -0.121506236494 0.000324490472957 0.00800387765885\n",
      "173 40 -0.0691983774304 0.000324490472957 0.0080043756278\n",
      "173 45 -0.0485810935497 0.000324490472957 0.0080049848806\n",
      "173 50 -0.0637694597244 0.000324490472957 0.00800554644363\n",
      "173 55 -0.0569141022861 0.000324490472957 0.00800627314548\n",
      "valid_acc 98.00166666666667\n",
      "174 0 -0.0567148439586 0.000324490472957 0.00800694926781\n",
      "174 5 -0.0522656776011 0.000324490472957 0.00800744675477\n",
      "174 10 -0.0602045655251 0.000324490472957 0.00800775648997\n",
      "174 15 -0.0702563077211 0.000324490472957 0.00800846971056\n",
      "174 20 -0.0875075682998 0.000324490472957 0.00800942604908\n",
      "174 25 -0.0724499821663 0.000324490472957 0.00801045762103\n",
      "174 30 -0.0561317875981 0.000324490472957 0.00801159320913\n",
      "174 35 -0.0683006122708 0.000324490472957 0.00801270686646\n",
      "174 40 -0.0496231354773 0.000324490472957 0.00801379231802\n",
      "174 45 -0.0555670000613 0.000324490472957 0.00801483305582\n",
      "174 50 -0.0604318007827 0.000324490472957 0.00801577337821\n",
      "174 55 -0.0733899101615 0.000324490472957 0.00801668979724\n",
      "valid_acc 98.0\n",
      "175 0 -0.0696281120181 0.000324490472957 0.00801761271594\n",
      "175 5 -0.0782447829843 0.000324490472957 0.0080185557091\n",
      "175 10 -0.0390397422016 0.000324490472957 0.00801976009514\n",
      "175 15 -0.0489281602204 0.000324490472957 0.00802088351381\n",
      "175 20 -0.0804003477097 0.000324490472957 0.00802168053293\n",
      "175 25 -0.0693043842912 0.000324490472957 0.00802239472501\n",
      "175 30 -0.0787957310677 0.000324490472957 0.00802318393803\n",
      "175 35 -0.0572490394115 0.000324490472957 0.00802372016783\n",
      "175 40 -0.0753127261996 0.000324490472957 0.00802413410809\n",
      "175 45 -0.067791081965 0.000324490472957 0.00802453139967\n",
      "175 50 -0.0827506706119 0.000324490472957 0.00802495122222\n",
      "175 55 -0.0696682557464 0.000324490472957 0.00802546527726\n",
      "valid_acc 97.98666666666666\n",
      "176 0 -0.0460725128651 0.000324490472957 0.00802604834141\n",
      "176 5 -0.0597245283425 0.000324490472957 0.00802655272199\n",
      "176 10 -0.0741751939058 0.000324490472957 0.00802704544782\n",
      "176 15 -0.0539204962552 0.000324490472957 0.00802750618789\n",
      "176 20 -0.0479907244444 0.000324490472957 0.00802804465102\n",
      "176 25 -0.0549919754267 0.000324490472957 0.00802862908455\n",
      "176 30 -0.0692070052028 0.000324490472957 0.00802908007047\n",
      "176 35 -0.0760443136096 0.000324490472957 0.00802941981556\n",
      "176 40 -0.0400595329702 0.000324490472957 0.00802954164151\n",
      "176 45 -0.0717367976904 0.000324490472957 0.00802950169102\n",
      "176 50 -0.0806963518262 0.000324490472957 0.00802922162852\n",
      "176 55 -0.0874940231442 0.000324490472957 0.00802863686992\n",
      "valid_acc 98.00333333333333\n",
      "177 0 -0.0519650988281 0.000324490472957 0.00802788483496\n",
      "177 5 -0.0701598152518 0.000324490472957 0.00802714287743\n",
      "177 10 -0.0623462609947 0.000324490472957 0.00802647031068\n",
      "177 15 -0.0460800230503 0.000324490472957 0.00802576436495\n",
      "177 20 -0.0672207400203 0.000324490472957 0.00802520139292\n",
      "177 25 -0.0702172368765 0.000324490472957 0.00802483209304\n",
      "177 30 -0.0365757644176 0.000324490472957 0.00802430252831\n",
      "177 35 -0.0665670633316 0.000324490472957 0.0080236409631\n",
      "177 40 -0.0620280578732 0.000324490472957 0.00802290254265\n",
      "177 45 -0.041912343353 0.000324490472957 0.00802193420732\n",
      "177 50 -0.045918289572 0.000324490472957 0.0080209252063\n",
      "177 55 -0.0675870999694 0.000324490472957 0.00802007451234\n",
      "valid_acc 97.94333333333334\n",
      "178 0 -0.0472286678851 0.000324490472957 0.00801934016367\n",
      "178 5 -0.0688709244132 0.000324490472957 0.00801859158664\n",
      "178 10 -0.0586951524019 0.000324490472957 0.00801768302947\n",
      "178 15 -0.0755510106683 0.000324490472957 0.00801685965533\n",
      "178 20 -0.0310085266829 0.000324490472957 0.00801587587218\n",
      "178 25 -0.0677653849125 0.000324490472957 0.00801479531985\n",
      "178 30 -0.0707826092839 0.000324490472957 0.00801369868615\n",
      "178 35 -0.0690957605839 0.000324490472957 0.0080123669608\n",
      "178 40 -0.0588269159198 0.000324490472957 0.00801083214069\n",
      "178 45 -0.064842492342 0.000324490472957 0.00800943100086\n",
      "178 50 -0.0431383103132 0.000324490472957 0.00800829676045\n",
      "178 55 -0.0532714836299 0.000324490472957 0.00800712541168\n",
      "valid_acc 98.03666666666668\n",
      "best valid_acc 98.03666666666668\n",
      "179 0 -0.0588778592646 0.000324490472957 0.00800594004069\n",
      "179 5 -0.0646199584007 0.000324490472957 0.00800471189297\n",
      "179 10 -0.0341973043978 0.000324490472957 0.00800319091705\n",
      "179 15 -0.0481130890548 0.000324490472957 0.00800154318257\n",
      "179 20 -0.059473015368 0.000324490472957 0.00800000606022\n",
      "179 25 -0.0563597865403 0.000324490472957 0.00799825157033\n",
      "179 30 -0.0659748092294 0.000324490472957 0.0079963408814\n",
      "179 35 -0.0787543132901 0.000324490472957 0.00799478480001\n",
      "179 40 -0.066293053329 0.000324490472957 0.0079934055534\n",
      "179 45 -0.0458808206022 0.000324490472957 0.00799192740458\n",
      "179 50 -0.079135902226 0.000324490472957 0.00799048323154\n",
      "179 55 -0.0504994876683 0.000324490472957 0.00798923503268\n",
      "valid_acc 98.03833333333334\n",
      "best valid_acc 98.03833333333334\n",
      "180 0 -0.0775743424892 0.000324490472957 0.00798809386298\n",
      "180 5 -0.0745970234275 0.000324490472957 0.00798678370575\n",
      "180 10 -0.0652909427881 0.000324490472957 0.0079855835435\n",
      "180 15 -0.044515196234 0.000324490472957 0.00798439546\n",
      "180 20 -0.0586160048842 0.000324490472957 0.00798327387501\n",
      "180 25 -0.0713494047523 0.000324490472957 0.00798221144895\n",
      "180 30 -0.0775178894401 0.000324490472957 0.00798137410048\n",
      "180 35 -0.0471819415689 0.000324490472957 0.00798049532936\n",
      "180 40 -0.0795161128044 0.000324490472957 0.00797977091978\n",
      "180 45 -0.069438368082 0.000324490472957 0.00797904801109\n",
      "180 50 -0.0754354670644 0.000324490472957 0.00797833246913\n",
      "180 55 -0.0556130520999 0.000324490472957 0.00797749336909\n",
      "valid_acc 97.975\n",
      "181 0 -0.0685410052538 0.000324490472957 0.00797662048065\n",
      "181 5 -0.0714367777109 0.000324490472957 0.00797560083433\n",
      "181 10 -0.0435633212328 0.000324490472957 0.00797460911013\n",
      "181 15 -0.059685587883 0.000324490472957 0.00797357074665\n",
      "181 20 -0.0693230181932 0.000324490472957 0.00797233158483\n",
      "181 25 -0.039006549865 0.000324490472957 0.00797106755781\n",
      "181 30 -0.0600183904171 0.000324490472957 0.00796982838841\n",
      "181 35 -0.0698880329728 0.000324490472957 0.00796850676614\n",
      "181 40 -0.0776318907738 0.000324490472957 0.00796696805274\n",
      "181 45 -0.0530385635793 0.000324490472957 0.00796523923846\n",
      "181 50 -0.0528300143778 0.000324490472957 0.00796350447505\n",
      "181 55 -0.0764250829816 0.000324490472957 0.00796152359336\n",
      "valid_acc 98.00666666666666\n",
      "182 0 -0.0714133530855 0.000324490472957 0.00795938205799\n",
      "182 5 -0.0728907361627 0.000324490472957 0.00795744915575\n",
      "182 10 -0.0526405014098 0.000324490472957 0.00795548534879\n",
      "182 15 -0.0532377772033 0.000324490472957 0.00795354721044\n",
      "182 20 -0.0595861077309 0.000324490472957 0.00795152804875\n",
      "182 25 -0.0520025677979 0.000324490472957 0.00794958320754\n",
      "182 30 -0.0712475702167 0.000324490472957 0.00794781683187\n",
      "182 35 -0.0809565037489 0.000324490472957 0.00794612314249\n",
      "182 40 -0.0947340652347 0.000324490472957 0.00794453633652\n",
      "182 45 -0.0504987314343 0.000324490472957 0.00794288503641\n",
      "182 50 -0.050295483321 0.000324490472957 0.00794116992198\n",
      "182 55 -0.0464959591627 0.000324490472957 0.00793955722251\n",
      "valid_acc 98.04166666666667\n",
      "best valid_acc 98.04166666666667\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "183 0 -0.0857739523053 0.000324490472957 0.00793800526956\n",
      "183 5 -0.0598147362471 0.000324490472957 0.00793620574177\n",
      "183 10 -0.0954408422112 0.000324490472957 0.00793432361642\n",
      "183 15 -0.0559049658477 0.000324490472957 0.00793234219828\n",
      "183 20 -0.0548945330083 0.000324490472957 0.00793034869321\n",
      "183 25 -0.0412732809782 0.000324490472957 0.00792830989778\n",
      "183 30 -0.059975836426 0.000324490472957 0.00792622523744\n",
      "183 35 -0.0862057879567 0.000324490472957 0.00792413163954\n",
      "183 40 -0.0619197785854 0.000324490472957 0.00792206715155\n",
      "183 45 -0.0425627455115 0.000324490472957 0.00791999963469\n",
      "183 50 -0.0395263954997 0.000324490472957 0.00791777626227\n",
      "183 55 -0.0493028797209 0.000324490472957 0.00791542772652\n",
      "valid_acc 98.05\n",
      "best valid_acc 98.05\n",
      "184 0 -0.0661923065782 0.000324490472957 0.00791295390579\n",
      "184 5 -0.0463431030512 0.000324490472957 0.00791047345527\n",
      "184 10 -0.0736695304513 0.000324490472957 0.00790797159123\n",
      "184 15 -0.038525596261 0.000324490472957 0.00790540526208\n",
      "184 20 -0.0582977607846 0.000324490472957 0.00790279865781\n",
      "184 25 -0.0597551316023 0.000324490472957 0.00790019092424\n",
      "184 30 -0.0510069355369 0.000324490472957 0.0078978039081\n",
      "184 35 -0.0607289597392 0.000324490472957 0.00789564731534\n",
      "184 40 -0.0489901155233 0.000324490472957 0.00789350750275\n",
      "184 45 -0.0632219985127 0.000324490472957 0.00789133059366\n",
      "184 50 -0.0718220025301 0.000324490472957 0.00788895110865\n",
      "184 55 -0.0505528971553 0.000324490472957 0.00788664108891\n",
      "valid_acc 98.05833333333334\n",
      "best valid_acc 98.05833333333334\n",
      "185 0 -0.0570363178849 0.000324490472957 0.00788428399982\n",
      "185 5 -0.075536698103 0.000324490472957 0.00788207520694\n",
      "185 10 -0.0384400486946 0.000324490472957 0.00787994300678\n",
      "185 15 -0.076409868896 0.000324490472957 0.00787786151996\n",
      "185 20 -0.0396604314446 0.000324490472957 0.00787593020578\n",
      "185 25 -0.0729097276926 0.000324490472957 0.00787412462224\n",
      "185 30 -0.0831876993179 0.000324490472957 0.00787225701146\n",
      "185 35 -0.0679722875357 0.000324490472957 0.00787029046114\n",
      "185 40 -0.0549504682422 0.000324490472957 0.00786820206854\n",
      "185 45 -0.059604305774 0.000324490472957 0.00786624748507\n",
      "185 50 -0.0494408234954 0.000324490472957 0.00786454135185\n",
      "185 55 -0.0471522435546 0.000324490472957 0.00786284498637\n",
      "valid_acc 98.08666666666667\n",
      "best valid_acc 98.08666666666667\n",
      "186 0 -0.0707256272435 0.000324490472957 0.00786113034538\n",
      "186 5 -0.0707315430045 0.000324490472957 0.00785950501221\n",
      "186 10 -0.0600496381521 0.000324490472957 0.00785787892839\n",
      "186 15 -0.0718142017722 0.000324490472957 0.00785636061277\n",
      "186 20 -0.0498154573143 0.000324490472957 0.00785494235209\n",
      "186 25 -0.0672348886728 0.000324490472957 0.00785341936209\n",
      "186 30 -0.0874228999019 0.000324490472957 0.00785202274031\n",
      "186 35 -0.0463581122458 0.000324490472957 0.00785039643994\n",
      "186 40 -0.0537263192236 0.000324490472957 0.00784865173412\n",
      "186 45 -0.0496253743768 0.000324490472957 0.00784684427967\n",
      "186 50 -0.0499134808779 0.000324490472957 0.00784517796973\n",
      "186 55 -0.034596260637 0.000324490472957 0.00784343551536\n",
      "valid_acc 98.06333333333333\n",
      "187 0 -0.0763726904988 0.000324490472957 0.00784171357153\n",
      "187 5 -0.0532067939639 0.000324490472957 0.00784018697956\n",
      "187 10 -0.052163656801 0.000324490472957 0.00783885353085\n",
      "187 15 -0.0534307211637 0.000324490472957 0.00783773979028\n",
      "187 20 -0.0725234523416 0.000324490472957 0.0078366452357\n",
      "187 25 -0.0560684800148 0.000324490472957 0.00783544057211\n",
      "187 30 -0.0617341063917 0.000324490472957 0.00783416611925\n",
      "187 35 -0.0627831891179 0.000324490472957 0.00783301227413\n",
      "187 40 -0.0615451075137 0.000324490472957 0.00783192042357\n",
      "187 45 -0.0637919679284 0.000324490472957 0.00783094086168\n",
      "187 50 -0.0791485309601 0.000324490472957 0.00783020884816\n",
      "187 55 -0.0510006807745 0.000324490472957 0.00782959278517\n",
      "valid_acc 98.09166666666667\n",
      "best valid_acc 98.09166666666667\n",
      "188 0 -0.0623120963573 0.000324490472957 0.00782913053973\n",
      "188 5 -0.0798293203115 0.000324490472957 0.00782868568917\n",
      "188 10 -0.040358543396 0.000324490472957 0.00782819576843\n",
      "188 15 -0.0739649832249 0.000324490472957 0.00782749778424\n",
      "188 20 -0.0293639246374 0.000324490472957 0.00782687604302\n",
      "188 25 -0.0691125616431 0.000324490472957 0.00782620692547\n",
      "188 30 -0.0448346324265 0.000324490472957 0.007825633713\n",
      "188 35 -0.0481851957738 0.000324490472957 0.00782503981256\n",
      "188 40 -0.0594095550478 0.000324490472957 0.00782439332674\n",
      "188 45 -0.0385766103864 0.000324490472957 0.00782382428306\n",
      "188 50 -0.059289611876 0.000324490472957 0.00782310095221\n",
      "188 55 -0.0698794648051 0.000324490472957 0.00782219477733\n",
      "valid_acc 98.00999999999999\n",
      "189 0 -0.0662537813187 0.000324490472957 0.00782135775758\n",
      "189 5 -0.0681081563234 0.000324490472957 0.00782053484286\n",
      "189 10 -0.07906422019 0.000324490472957 0.0078199890155\n",
      "189 15 -0.0487946867943 0.000324490472957 0.00781962596757\n",
      "189 20 -0.0823152288795 0.000324490472957 0.00781926282967\n",
      "189 25 -0.0762570947409 0.000324490472957 0.0078187830778\n",
      "189 30 -0.0871272161603 0.000324490472957 0.00781834328758\n",
      "189 35 -0.0370339676738 0.000324490472957 0.00781789899692\n",
      "189 40 -0.0485099032521 0.000324490472957 0.00781746035263\n",
      "189 45 -0.0464895814657 0.000324490472957 0.00781709793061\n",
      "189 50 -0.0581821724772 0.000324490472957 0.00781687738908\n",
      "189 55 -0.0649196356535 0.000324490472957 0.00781663453518\n",
      "valid_acc 97.985\n",
      "190 0 -0.0629473850131 0.000324490472957 0.00781627002694\n",
      "190 5 -0.0572427324951 0.000324490472957 0.00781585664248\n",
      "190 10 -0.0490417778492 0.000324490472957 0.00781540199795\n",
      "190 15 -0.0615675114095 0.000324490472957 0.00781496238246\n",
      "190 20 -0.0709008350968 0.000324490472957 0.00781464254784\n",
      "190 25 -0.0477946847677 0.000324490472957 0.00781439191532\n",
      "190 30 -0.0429535917938 0.000324490472957 0.007814173514\n",
      "190 35 -0.0760181173682 0.000324490472957 0.00781416510237\n",
      "190 40 -0.0657454356551 0.000324490472957 0.0078141109204\n",
      "190 45 -0.0677546858788 0.000324490472957 0.00781412863718\n",
      "190 50 -0.0340559706092 0.000324490472957 0.00781408384278\n",
      "190 55 -0.0478813536465 0.000324490472957 0.00781414665393\n",
      "valid_acc 98.055\n",
      "191 0 -0.0829272121191 0.000324490472957 0.00781422840092\n",
      "191 5 -0.0838208645582 0.000324490472957 0.00781406456316\n",
      "191 10 -0.0665652602911 0.000324490472957 0.00781393519187\n",
      "191 15 -0.0623025260866 0.000324490472957 0.00781405001132\n",
      "191 20 -0.0639624223113 0.000324490472957 0.0078140348356\n",
      "191 25 -0.0554707981646 0.000324490472957 0.00781392226423\n",
      "191 30 -0.0484697967768 0.000324490472957 0.00781377781368\n",
      "191 35 -0.0431696623564 0.000324490472957 0.00781378035288\n",
      "191 40 -0.0493536815047 0.000324490472957 0.0078137760333\n",
      "191 45 -0.0596284456551 0.000324490472957 0.00781386039479\n",
      "191 50 -0.0713310316205 0.000324490472957 0.00781390258982\n",
      "191 55 -0.0764361917973 0.000324490472957 0.00781409197272\n",
      "valid_acc 98.09333333333333\n",
      "best valid_acc 98.09333333333333\n",
      "192 0 -0.0564498156309 0.000324490472957 0.00781451238548\n",
      "192 5 -0.0707699134946 0.000324490472957 0.00781499190708\n",
      "192 10 -0.0544455386698 0.000324490472957 0.00781548271927\n",
      "192 15 -0.0541015267372 0.000324490472957 0.00781580953869\n",
      "192 20 -0.0458335913718 0.000324490472957 0.00781611603791\n",
      "192 25 -0.0458370931447 0.000324490472957 0.00781630456457\n",
      "192 30 -0.0503257624805 0.000324490472957 0.00781663537336\n",
      "192 35 -0.0690434575081 0.000324490472957 0.00781704171343\n",
      "192 40 -0.0607509203255 0.000324490472957 0.00781754516469\n",
      "192 45 -0.0846493318677 0.000324490472957 0.00781801298697\n",
      "192 50 -0.0522737838328 0.000324490472957 0.0078184425743\n",
      "192 55 -0.058548476547 0.000324490472957 0.00781888306676\n",
      "valid_acc 98.115\n",
      "best valid_acc 98.115\n",
      "193 0 -0.0391212143004 0.000324490472957 0.00781929950358\n",
      "193 5 -0.0530145540833 0.000324490472957 0.00781968114096\n",
      "193 10 -0.0578589253128 0.000324490472957 0.00781983834668\n",
      "193 15 -0.0670198872685 0.000324490472957 0.00781984923955\n",
      "193 20 -0.0475432910025 0.000324490472957 0.00781965977819\n",
      "193 25 -0.0668985769153 0.000324490472957 0.00781958427094\n",
      "193 30 -0.0605669654906 0.000324490472957 0.0078194514395\n",
      "193 35 -0.0398314967752 0.000324490472957 0.00781934702852\n",
      "193 40 -0.0533156581223 0.000324490472957 0.00781942422135\n",
      "193 45 -0.0783950388432 0.000324490472957 0.00781970862547\n",
      "193 50 -0.0628340244293 0.000324490472957 0.00782000727301\n",
      "193 55 -0.0723662674427 0.000324490472957 0.00782021584486\n",
      "valid_acc 98.07666666666667\n",
      "194 0 -0.0609648413956 0.000324490472957 0.00782060411329\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "194 5 -0.052438031882 0.000324490472957 0.00782106327081\n",
      "194 10 -0.0669089183211 0.000324490472957 0.00782159040831\n",
      "194 15 -0.0401438064873 0.000324490472957 0.00782213819233\n",
      "194 20 -0.0617402344942 0.000324490472957 0.00782266203615\n",
      "194 25 -0.0577086657286 0.000324490472957 0.00782321884072\n",
      "194 30 -0.0473446212709 0.000324490472957 0.00782355479096\n",
      "194 35 -0.0422920621932 0.000324490472957 0.0078239272085\n",
      "194 40 -0.0581905990839 0.000324490472957 0.0078244520323\n",
      "194 45 -0.0475893653929 0.000324490472957 0.00782503261346\n",
      "194 50 -0.0686567276716 0.000324490472957 0.00782550447509\n",
      "194 55 -0.0763478130102 0.000324490472957 0.00782566913629\n",
      "valid_acc 98.01666666666667\n",
      "195 0 -0.0725594609976 0.000324490472957 0.00782568177993\n",
      "195 5 -0.0768280848861 0.000324490472957 0.00782572126985\n",
      "195 10 -0.0503028407693 0.000324490472957 0.00782580632468\n",
      "195 15 -0.0607126951218 0.000324490472957 0.00782578992301\n",
      "195 20 -0.0670951828361 0.000324490472957 0.00782592686714\n",
      "195 25 -0.0368602611125 0.000324490472957 0.00782629604125\n",
      "195 30 -0.0439495332539 0.000324490472957 0.00782679821732\n",
      "195 35 -0.074712112546 0.000324490472957 0.00782720388872\n",
      "195 40 -0.0342166572809 0.000324490472957 0.0078274992757\n",
      "195 45 -0.0556475929916 0.000324490472957 0.00782787109807\n",
      "195 50 -0.0777353420854 0.000324490472957 0.00782820477068\n",
      "195 55 -0.0858059450984 0.000324490472957 0.00782858711393\n",
      "valid_acc 98.02333333333333\n",
      "196 0 -0.0359502956271 0.000324490472957 0.00782887112724\n",
      "196 5 -0.0782997906208 0.000324490472957 0.00782916361635\n",
      "196 10 -0.0441711954772 0.000324490472957 0.00782955250012\n",
      "196 15 -0.0318052880466 0.000324490472957 0.00782994052184\n",
      "196 20 -0.0773058459163 0.000324490472957 0.0078302214893\n",
      "196 25 -0.0617293082178 0.000324490472957 0.00783046493996\n",
      "196 30 -0.0419329926372 0.000324490472957 0.00783078730677\n",
      "196 35 -0.0756485238671 0.000324490472957 0.00783094033065\n",
      "196 40 -0.0890170112252 0.000324490472957 0.00783112903612\n",
      "196 45 -0.0411653108895 0.000324490472957 0.00783128703671\n",
      "196 50 -0.077457152307 0.000324490472957 0.00783154748111\n",
      "196 55 -0.0845699906349 0.000324490472957 0.00783192077945\n",
      "valid_acc 98.08666666666667\n",
      "197 0 -0.0581255815923 0.000324490472957 0.00783234179231\n",
      "197 5 -0.0549236238003 0.000324490472957 0.00783287980223\n",
      "197 10 -0.069108851254 0.000324490472957 0.00783335201674\n",
      "197 15 -0.0799346789718 0.000324490472957 0.00783389847537\n",
      "197 20 -0.0379017367959 0.000324490472957 0.00783427231422\n",
      "197 25 -0.0578072965145 0.000324490472957 0.00783448027139\n",
      "197 30 -0.0737811923027 0.000324490472957 0.00783470639895\n",
      "197 35 -0.0656832903624 0.000324490472957 0.00783505172267\n",
      "197 40 -0.0702839270234 0.000324490472957 0.0078353835998\n",
      "197 45 -0.0733957141638 0.000324490472957 0.00783576070572\n",
      "197 50 -0.0609378367662 0.000324490472957 0.00783629226554\n",
      "197 55 -0.0591236650944 0.000324490472957 0.00783704369955\n",
      "valid_acc 97.995\n",
      "198 0 -0.0910415574908 0.000324490472957 0.00783793045528\n",
      "198 5 -0.0892538055778 0.000324490472957 0.0078387569385\n",
      "198 10 -0.0539416149259 0.000324490472957 0.0078396340527\n",
      "198 15 -0.0859425067902 0.000324490472957 0.00784050685164\n",
      "198 20 -0.0483903326094 0.000324490472957 0.00784124471695\n",
      "198 25 -0.0542340241373 0.000324490472957 0.00784182236566\n",
      "198 30 -0.0373716205359 0.000702759599916 0.00784259695041\n",
      "198 35 -0.0360180586576 0.000702759599916 0.00784344022655\n",
      "198 40 -0.0844640657306 0.000702759599916 0.00784436129862\n",
      "198 45 -0.0676791220903 0.000702759599916 0.00784537018834\n",
      "198 50 -0.0728174224496 0.000702759599916 0.00784643751829\n",
      "198 55 -0.0672119185328 0.000702759599916 0.00784780545521\n",
      "valid_acc 97.99166666666666\n",
      "199 0 -0.0636923834682 0.000702759599916 0.007849475373\n",
      "199 5 -0.0826262682676 0.000702759599916 0.00785110341706\n",
      "199 10 -0.0558455549181 0.000702759599916 0.00785271592464\n",
      "199 15 -0.0510112047195 0.000702759599916 0.00785423403739\n",
      "199 20 -0.0973582491279 0.000702759599916 0.0078557179463\n",
      "199 25 -0.0420454442501 0.000702759599916 0.00785721926819\n",
      "199 30 -0.0777081549168 0.000702759599916 0.00785869660457\n",
      "199 35 -0.0818355605006 0.000702759599916 0.00786025381682\n",
      "199 40 -0.116208344698 0.000702759599916 0.0078617659564\n",
      "199 45 -0.046775855124 0.000702759599916 0.00786332759086\n",
      "199 50 -0.0553469136357 0.000702759599916 0.00786489225383\n",
      "199 55 -0.0631591379642 0.000702759599916 0.0078667326789\n",
      "valid_acc 98.01333333333334\n",
      "200 0 -0.0565894991159 0.000702759599916 0.00786860826869\n",
      "200 5 -0.0446275286376 0.000702759599916 0.00787056993512\n",
      "200 10 -0.0496098175645 0.000702759599916 0.00787247684535\n",
      "200 15 -0.0679839923978 0.000702759599916 0.00787445080256\n",
      "200 20 -0.0687419772148 0.000702759599916 0.00787618015083\n",
      "200 25 -0.0525558963418 0.000702759599916 0.00787779983466\n",
      "200 30 -0.0833660811186 0.000702759599916 0.00787924660807\n",
      "200 35 -0.0740512013435 0.000702759599916 0.00788065892246\n",
      "200 40 -0.0716491863132 0.000702759599916 0.0078819579961\n",
      "200 45 -0.0467267110944 0.000702759599916 0.00788320569872\n",
      "200 50 -0.0470679551363 0.000702759599916 0.00788436297518\n",
      "200 55 -0.0514583326876 0.000702759599916 0.00788537118006\n",
      "valid_acc 98.08333333333333\n",
      "201 0 -0.0502796471119 0.000702759599916 0.00788625976971\n",
      "201 5 -0.060273565352 0.000702759599916 0.0078872767282\n",
      "201 10 -0.0572499521077 0.000702759599916 0.00788837346063\n",
      "201 15 -0.0694478154182 0.000702759599916 0.00788950627099\n",
      "201 20 -0.0459824614227 0.000702759599916 0.00789062560073\n",
      "201 25 -0.0597688928246 0.000702759599916 0.00789164211097\n",
      "201 30 -0.0579339116812 0.000702759599916 0.00789271656689\n",
      "201 35 -0.057163272053 0.000702759599916 0.00789367884155\n",
      "201 40 -0.0785678103566 0.000702759599916 0.00789448368166\n",
      "201 45 -0.0492488332093 0.000702759599916 0.00789503407561\n",
      "201 50 -0.0401133336127 0.000702759599916 0.00789569605009\n",
      "201 55 -0.0904376059771 0.000702759599916 0.00789631377418\n",
      "valid_acc 98.07000000000001\n",
      "202 0 -0.0473025068641 0.000702759599916 0.00789688820841\n",
      "202 5 -0.0531685389578 0.000702759599916 0.00789731224948\n",
      "202 10 -0.056128975004 0.000702759599916 0.00789774927848\n",
      "202 15 -0.0393397696316 0.000702759599916 0.00789823110714\n",
      "202 20 -0.0683078989387 0.000702759599916 0.00789874419497\n",
      "202 25 -0.0667815953493 0.000702759599916 0.00789933143981\n",
      "202 30 -0.0620913505554 0.000702759599916 0.00790012235911\n",
      "202 35 -0.050682310015 0.000702759599916 0.00790090436778\n",
      "202 40 -0.0685095638037 0.000702759599916 0.00790154191038\n",
      "202 45 -0.058499481529 0.000702759599916 0.00790218987632\n",
      "202 50 -0.0583033747971 0.000702759599916 0.00790271354901\n",
      "202 55 -0.0575478971004 0.000702759599916 0.0079031288774\n",
      "valid_acc 98.07000000000001\n",
      "203 0 -0.0437286496162 0.000702759599916 0.00790348264946\n",
      "203 5 -0.0545810312033 0.000702759599916 0.00790373331334\n",
      "203 10 -0.0715518593788 0.000702759599916 0.00790395217895\n",
      "203 15 -0.0738238617778 0.000702759599916 0.00790405017554\n",
      "203 20 -0.0557114854455 0.000702759599916 0.00790395886286\n",
      "203 25 -0.054541438818 0.000702759599916 0.00790383162788\n",
      "203 30 -0.040187984705 0.000702759599916 0.00790359109592\n",
      "203 35 -0.0557809919119 0.000702759599916 0.00790302143674\n",
      "203 40 -0.091770388186 0.000702759599916 0.0079024858506\n",
      "203 45 -0.0518694035709 0.000702759599916 0.00790195444502\n",
      "203 50 -0.0456523783505 0.000702759599916 0.00790130220579\n",
      "203 55 -0.0714587941766 0.000702759599916 0.00790088987073\n",
      "valid_acc 98.05\n",
      "204 0 -0.0648954361677 0.000702759599916 0.00790050827152\n",
      "204 5 -0.052481751889 0.000702759599916 0.00790005995198\n",
      "204 10 -0.0696916207671 0.000702759599916 0.00789960102288\n",
      "204 15 -0.0445906817913 0.000702759599916 0.00789906898699\n",
      "204 20 -0.0678629204631 0.000702759599916 0.00789846770822\n",
      "204 25 -0.0465532280505 0.000702759599916 0.00789783757478\n",
      "204 30 -0.0748662427068 0.000702759599916 0.00789725702744\n",
      "204 35 -0.0808694809675 0.000702759599916 0.00789658979565\n",
      "204 40 -0.0738730356097 0.000702759599916 0.00789601505103\n",
      "204 45 -0.0417492315173 0.000702759599916 0.00789543875147\n",
      "204 50 -0.0652088373899 0.000702759599916 0.00789489512683\n",
      "204 55 -0.0408621579409 0.000702759599916 0.00789435275822\n",
      "valid_acc 98.055\n",
      "205 0 -0.0315160304308 0.000702759599916 0.00789374928572\n",
      "205 5 -0.0852884352207 0.000702759599916 0.00789342512355\n",
      "205 10 -0.0690926015377 0.000702759599916 0.00789307416541\n",
      "205 15 -0.0462497845292 0.000702759599916 0.00789281944586\n",
      "205 20 -0.0514484345913 0.000702759599916 0.00789248355822\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "205 25 -0.0644433870912 0.000702759599916 0.00789214551282\n",
      "205 30 -0.0457796491683 0.000702759599916 0.00789184520365\n",
      "205 35 -0.0577623583376 0.000702759599916 0.00789155260403\n",
      "205 40 -0.0567684173584 0.000702759599916 0.00789132577302\n",
      "205 45 -0.0527089983225 0.000702759599916 0.0078910955578\n",
      "205 50 -0.040148537606 0.000702759599916 0.00789088800284\n",
      "205 55 -0.0771073773503 0.000702759599916 0.00789068105832\n",
      "valid_acc 98.08333333333333\n",
      "206 0 -0.0635924786329 0.000702759599916 0.00789031850223\n",
      "206 5 -0.0551059283316 0.000702759599916 0.0078901873253\n",
      "206 10 -0.0752752497792 0.000702759599916 0.00788989828368\n",
      "206 15 -0.0479792617261 0.000702759599916 0.00788957663703\n",
      "206 20 -0.056997179985 0.000702759599916 0.0078892501712\n",
      "206 25 -0.0542194880545 0.000702759599916 0.00788880400566\n",
      "206 30 -0.053319003433 0.000702759599916 0.00788838853913\n",
      "206 35 -0.048369910568 0.000702759599916 0.00788786969703\n",
      "206 40 -0.0615018382668 0.000702759599916 0.00788737490259\n",
      "206 45 -0.0427993163466 0.000702759599916 0.00788678711881\n",
      "206 50 -0.056518651545 0.000702759599916 0.00788608665034\n",
      "206 55 -0.0685337856412 0.000702759599916 0.00788531590814\n",
      "valid_acc 98.08833333333334\n",
      "207 0 -0.0519077293575 0.000702759599916 0.00788443851856\n",
      "207 5 -0.0471961833537 0.000702759599916 0.0078834966287\n",
      "207 10 -0.0429756678641 0.000702759599916 0.0078825384325\n",
      "207 15 -0.0428383313119 0.000702759599916 0.00788152579423\n",
      "207 20 -0.0675594732165 0.000702759599916 0.00788054869555\n",
      "207 25 -0.0436191037297 0.000702759599916 0.0078794944867\n",
      "207 30 -0.0579709410667 0.000702759599916 0.00787821498333\n",
      "207 35 -0.0960940867662 0.000702759599916 0.0078769589637\n",
      "207 40 -0.0430026911199 0.000702759599916 0.0078756615716\n",
      "207 45 -0.0611031241715 0.000702759599916 0.00787431648893\n",
      "207 50 -0.0440714098513 0.000702759599916 0.00787278554622\n",
      "207 55 -0.0462847836316 0.000702759599916 0.00787112569636\n",
      "valid_acc 98.05833333333334\n",
      "208 0 -0.0609765797853 0.000702759599916 0.00786955977122\n",
      "208 5 -0.0531304292381 0.000702759599916 0.00786805799115\n",
      "208 10 -0.0623066499829 0.000702759599916 0.00786678429302\n",
      "208 15 -0.0604836605489 0.000702759599916 0.00786551214896\n",
      "208 20 -0.0669120326638 0.000702759599916 0.00786430990187\n",
      "208 25 -0.0768961086869 0.000702759599916 0.00786331567108\n",
      "208 30 -0.0733447596431 0.000702759599916 0.00786234796828\n",
      "208 35 -0.0412042625248 0.000702759599916 0.0078614353462\n",
      "208 40 -0.0526921376586 0.000702759599916 0.00786046754511\n",
      "208 45 -0.0725533440709 0.000702759599916 0.00785959575176\n",
      "208 50 -0.0534657165408 0.000702759599916 0.00785878835296\n",
      "208 55 -0.0654491633177 0.000702759599916 0.00785808887232\n",
      "valid_acc 98.13166666666666\n",
      "best valid_acc 98.13166666666666\n",
      "209 0 -0.041648644954 0.000702759599916 0.00785744477077\n",
      "209 5 -0.0650783479214 0.000702759599916 0.00785676854677\n",
      "209 10 -0.0508733280003 0.000702759599916 0.00785584080044\n",
      "209 15 -0.0489970631897 0.000702759599916 0.00785470563073\n",
      "209 20 -0.0366936959326 0.000702759599916 0.00785346116882\n",
      "209 25 -0.0620214603841 0.000702759599916 0.0078521706603\n",
      "209 30 -0.0519762262702 0.000702759599916 0.00785101366499\n",
      "209 35 -0.0740964487195 0.000702759599916 0.00784979793451\n",
      "209 40 -0.0647990554571 0.000702759599916 0.0078485891772\n",
      "209 45 -0.0539812333882 0.000702759599916 0.00784750310294\n",
      "209 50 -0.0575902312994 0.000702759599916 0.00784664649301\n",
      "209 55 -0.0414974465966 0.000702759599916 0.00784550742288\n",
      "valid_acc 98.11833333333333\n",
      "210 0 -0.0840002670884 0.000702759599916 0.00784406844197\n",
      "210 5 -0.0457446835935 0.000702759599916 0.00784267702397\n",
      "210 10 -0.0413249209523 0.000702759599916 0.0078410742943\n",
      "210 15 -0.050820954144 0.000702759599916 0.0078393796316\n",
      "210 20 -0.0490421764553 0.000702759599916 0.00783754469375\n",
      "210 25 -0.0306128151715 0.000702759599916 0.00783563982706\n",
      "210 30 -0.0647401213646 0.000702759599916 0.00783361446864\n",
      "210 35 -0.0489664152265 0.000702759599916 0.00783165229868\n",
      "210 40 -0.0736761465669 0.000702759599916 0.00782955225055\n",
      "210 45 -0.0526909828186 0.000702759599916 0.00782733728644\n",
      "210 50 -0.0479719303548 0.000702759599916 0.00782507332803\n",
      "210 55 -0.0508233718574 0.000702759599916 0.00782292023002\n",
      "valid_acc 98.08833333333334\n",
      "211 0 -0.0778239145875 0.000702759599916 0.00782076431392\n",
      "211 5 -0.0501849539578 0.000702759599916 0.00781846878722\n",
      "211 10 -0.0692232623696 0.000702759599916 0.00781617859064\n",
      "211 15 -0.054382994771 0.000702759599916 0.00781384679987\n",
      "211 20 -0.0599397085607 0.000702759599916 0.00781171017741\n",
      "211 25 -0.0837938636541 0.000702759599916 0.00780964568648\n",
      "211 30 -0.0672584548593 0.000702759599916 0.00780757462553\n",
      "211 35 -0.0480018928647 0.000702759599916 0.00780542186361\n",
      "211 40 -0.0424277409911 0.000702759599916 0.00780311535959\n",
      "211 45 -0.0553778931499 0.000702759599916 0.00780078280253\n",
      "211 50 -0.0741383805871 0.000702759599916 0.00779846535879\n",
      "211 55 -0.0614068321884 0.000702759599916 0.0077961680363\n",
      "valid_acc 98.13666666666667\n",
      "best valid_acc 98.13666666666667\n",
      "212 0 -0.0521343126893 0.000702759599916 0.00779365110397\n",
      "212 5 -0.0596551299095 0.000702759599916 0.00779106788374\n",
      "212 10 -0.0720146745443 0.000702759599916 0.00778836420085\n",
      "212 15 -0.062213037163 0.000702759599916 0.00778562034834\n",
      "212 20 -0.0435292832553 0.000702759599916 0.0077829147789\n",
      "212 25 -0.0480914004147 0.000702759599916 0.0077801691038\n",
      "212 30 -0.0620257817209 0.000702759599916 0.00777733968114\n",
      "212 35 -0.0634391307831 0.000702759599916 0.00777459232288\n",
      "212 40 -0.0836768895388 0.000702759599916 0.00777198530163\n",
      "212 45 -0.039525449276 0.000702759599916 0.00776945761749\n",
      "212 50 -0.0415913276374 0.000702759599916 0.00776696076439\n",
      "212 55 -0.0827936828136 0.000702759599916 0.00776448578022\n",
      "valid_acc 98.13666666666667\n",
      "best valid_acc 98.13666666666667\n",
      "213 0 -0.0747810080647 0.000702759599916 0.00776202744021\n",
      "213 5 -0.0414436496794 0.000702759599916 0.007759576333\n",
      "213 10 -0.052418101579 0.000702759599916 0.00775697111131\n",
      "213 15 -0.0506119914353 0.000702759599916 0.00775422163833\n",
      "213 20 -0.0603211037815 0.000702759599916 0.0077515649812\n",
      "213 25 -0.0432672277093 0.000702759599916 0.00774880630805\n",
      "213 30 -0.0719376653433 0.000702759599916 0.00774602862451\n",
      "213 35 -0.0582479536533 0.000702759599916 0.00774333387789\n",
      "213 40 -0.0590963661671 0.000702759599916 0.00774085617898\n",
      "213 45 -0.0571465566754 0.000702759599916 0.00773847628748\n",
      "213 50 -0.0449944995344 0.000702759599916 0.00773620137048\n",
      "213 55 -0.0580760240555 0.000702759599916 0.00773416360297\n",
      "valid_acc 98.11666666666666\n",
      "214 0 -0.0657234862447 0.000702759599916 0.00773221290651\n",
      "214 5 -0.0648208484054 0.000702759599916 0.00773029044217\n",
      "214 10 -0.0759300589561 0.000702759599916 0.00772830595602\n",
      "214 15 -0.0645362734795 0.000702759599916 0.0077262160488\n",
      "214 20 -0.0462273620069 0.000702759599916 0.00772433271564\n",
      "214 25 -0.0470681488514 0.000702759599916 0.00772271273827\n",
      "214 30 -0.0407307818532 0.000702759599916 0.00772122598121\n",
      "214 35 -0.029476787895 0.000702759599916 0.00771977574388\n",
      "214 40 -0.0384996235371 0.000702759599916 0.00771829493833\n",
      "214 45 -0.0587992705405 0.000702759599916 0.00771677981068\n",
      "214 50 -0.047083992511 0.000702759599916 0.00771528359632\n",
      "214 55 -0.0551621317863 0.000702759599916 0.00771371993537\n",
      "valid_acc 98.15833333333333\n",
      "best valid_acc 98.15833333333333\n",
      "215 0 -0.03591023013 0.000702759599916 0.0077120253059\n",
      "215 5 -0.0436335429549 0.000702759599916 0.00771039668104\n",
      "215 10 -0.0530452504754 0.000702759599916 0.00770877228879\n",
      "215 15 -0.0652144104242 0.000702759599916 0.00770727230828\n",
      "215 20 -0.0606914572418 0.000702759599916 0.00770612135465\n",
      "215 25 -0.0918887257576 0.000702759599916 0.00770512454005\n",
      "215 30 -0.0508704744279 0.000702759599916 0.00770430433864\n",
      "215 35 -0.0361027903855 0.000702759599916 0.007703478868\n",
      "215 40 -0.0463653318584 0.000702759599916 0.00770238964919\n",
      "215 45 -0.0556656494737 0.000702759599916 0.00770138570802\n",
      "215 50 -0.059660423547 0.000702759599916 0.00770042984973\n",
      "215 55 -0.0414119176567 0.000702759599916 0.00769941122857\n",
      "valid_acc 98.13833333333334\n",
      "216 0 -0.0577225498855 0.000702759599916 0.00769841240157\n",
      "216 5 -0.0599637068808 0.000702759599916 0.00769765326906\n",
      "216 10 -0.0600629523396 0.000702759599916 0.00769697660586\n",
      "216 15 -0.0457916110754 0.000702759599916 0.00769632884895\n",
      "216 20 -0.098956733942 0.000702759599916 0.00769556491015\n",
      "216 25 -0.0381135903299 0.000702759599916 0.00769506975987\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "216 30 -0.0557963885367 0.000702759599916 0.00769458794926\n",
      "216 35 -0.0515142343938 0.000702759599916 0.00769412834161\n",
      "216 40 -0.0520944036543 0.000702759599916 0.00769374316394\n",
      "216 45 -0.057604059577 0.000702759599916 0.00769329555915\n",
      "216 50 -0.0710696130991 0.000702759599916 0.00769280580342\n",
      "216 55 -0.0562067702413 0.000702759599916 0.00769217796266\n",
      "valid_acc 98.21833333333333\n",
      "best valid_acc 98.21833333333333\n",
      "217 0 -0.0572662390769 0.000702759599916 0.00769161830416\n",
      "217 5 -0.0601053349674 0.000702759599916 0.00769122079191\n",
      "217 10 -0.0582150407135 0.000702759599916 0.00769093047796\n",
      "217 15 -0.0605949126184 0.000702759599916 0.00769072818237\n",
      "217 20 -0.0805880129337 0.000702759599916 0.00769069256225\n",
      "217 25 -0.0598743483424 0.000702759599916 0.00769060135085\n",
      "217 30 -0.0623083598912 0.000702759599916 0.00769051609503\n",
      "217 35 -0.0658441483974 0.000702759599916 0.00769046141657\n",
      "217 40 -0.0613985061646 0.000702759599916 0.00769039339607\n",
      "217 45 -0.037593152374 0.000702759599916 0.00769029558442\n",
      "217 50 -0.0689451247454 0.000702759599916 0.0076902444908\n",
      "217 55 -0.0285050328821 0.000702759599916 0.00769019276279\n",
      "valid_acc 98.20833333333333\n",
      "218 0 -0.0689239278436 0.000702759599916 0.00768990578279\n",
      "218 5 -0.0444460585713 0.000702759599916 0.00768962360342\n",
      "218 10 -0.0578345917165 0.000702759599916 0.00768935200078\n",
      "218 15 -0.0541088022292 0.000702759599916 0.00768901001134\n",
      "218 20 -0.0483188517392 0.000702759599916 0.00768884586121\n",
      "218 25 -0.0765830054879 0.000702759599916 0.00768871557127\n",
      "218 30 -0.0822360664606 0.000702759599916 0.00768852563124\n",
      "218 35 -0.0415097475052 0.000702759599916 0.00768814927538\n",
      "218 40 -0.0689646229148 0.000702759599916 0.00768772395189\n",
      "218 45 -0.0498810522258 0.000702759599916 0.00768724650884\n",
      "218 50 -0.0458003133535 0.000702759599916 0.00768674984493\n",
      "218 55 -0.0573746822774 0.000702759599916 0.00768624651894\n",
      "valid_acc 98.14666666666668\n",
      "219 0 -0.0431525856256 0.000702759599916 0.00768581264179\n",
      "219 5 -0.0692056119442 0.000702759599916 0.00768559760339\n",
      "219 10 -0.0617468468845 0.000702759599916 0.00768550712675\n",
      "219 15 -0.0494638569653 0.000702759599916 0.0076855557616\n",
      "219 20 -0.0720158368349 0.000702759599916 0.00768564028706\n",
      "219 25 -0.0560019686818 0.000702759599916 0.00768570689452\n",
      "219 30 -0.0535717532039 0.000702759599916 0.00768568130825\n",
      "219 35 -0.0568982474506 0.000702759599916 0.0076855239663\n",
      "219 40 -0.0385798029602 0.000702759599916 0.00768528497611\n",
      "219 45 -0.0994533449411 0.000702759599916 0.00768494845844\n",
      "219 50 -0.0414039194584 0.000702759599916 0.0076846272072\n",
      "219 55 -0.06957924366 0.000702759599916 0.00768448167715\n",
      "valid_acc 98.11999999999999\n",
      "220 0 -0.0699832886457 0.000702759599916 0.00768421660232\n",
      "220 5 -0.0309220496565 0.000702759599916 0.00768399369575\n",
      "220 10 -0.0437547378242 0.000702759599916 0.00768368814029\n",
      "220 15 -0.0502321794629 0.000702759599916 0.00768336449673\n",
      "220 20 -0.0660962164402 0.000702759599916 0.00768294803823\n",
      "220 25 -0.0943779349327 0.000702759599916 0.00768249282918\n",
      "220 30 -0.0373748391867 0.000702759599916 0.00768192353405\n",
      "220 35 -0.0365678369999 0.000702759599916 0.00768122696897\n",
      "220 40 -0.0544244833291 0.000702759599916 0.00768035260694\n",
      "220 45 -0.0643813237548 0.000702759599916 0.00767942491014\n",
      "220 50 -0.0796868801117 0.000702759599916 0.00767865305893\n",
      "220 55 -0.0572502575815 0.000702759599916 0.00767794226006\n",
      "valid_acc 98.13666666666667\n",
      "221 0 -0.0903891995549 0.000702759599916 0.00767723937183\n",
      "221 5 -0.0377425998449 0.000702759599916 0.00767639656336\n",
      "221 10 -0.0615539364517 0.000702759599916 0.00767544786527\n",
      "221 15 -0.0810143947601 0.000702759599916 0.00767456544653\n",
      "221 20 -0.0527988485992 0.000702759599916 0.00767369553363\n",
      "221 25 -0.0639693588018 0.000702759599916 0.00767291952154\n",
      "221 30 -0.0719925910234 0.000702759599916 0.00767216109155\n",
      "221 35 -0.0332138948143 0.000702759599916 0.00767158828883\n",
      "221 40 -0.0403484590352 0.000702759599916 0.00767094613605\n",
      "221 45 -0.052101187408 0.000702759599916 0.00767029830491\n",
      "221 50 -0.0605575628579 0.000702759599916 0.00766962530945\n",
      "221 55 -0.0403701849282 0.000702759599916 0.0076690752046\n",
      "valid_acc 98.15333333333334\n",
      "222 0 -0.0571942478418 0.000702759599916 0.00766856973056\n",
      "222 5 -0.0618011020124 0.000702759599916 0.0076679166809\n",
      "222 10 -0.0425795763731 0.000702759599916 0.00766720175933\n",
      "222 15 -0.0518636070192 0.000702759599916 0.00766635378954\n",
      "222 20 -0.0617401562631 0.000702759599916 0.00766559995747\n",
      "222 25 -0.0673687681556 0.000702759599916 0.00766477328694\n",
      "222 30 -0.0430051162839 0.000702759599916 0.00766399390618\n",
      "222 35 -0.0623368285596 0.000702759599916 0.0076633825714\n",
      "222 40 -0.0595537610352 0.000702759599916 0.00766291114141\n",
      "222 45 -0.0598074421287 0.000702759599916 0.00766240631287\n",
      "222 50 -0.0668091922998 0.000702759599916 0.00766187712488\n",
      "222 55 -0.0382672175765 0.000702759599916 0.00766127292883\n",
      "valid_acc 98.13\n",
      "223 0 -0.0501725897193 0.000702759599916 0.00766059222984\n",
      "223 5 -0.0664390921593 0.000702759599916 0.0076600489354\n",
      "223 10 -0.0432500168681 0.000702759599916 0.0076596240746\n",
      "223 15 -0.0876936540008 0.000702759599916 0.00765942783709\n",
      "223 20 -0.0492151752114 0.000702759599916 0.00765898744927\n",
      "223 25 -0.0569759383798 0.000702759599916 0.00765837310412\n",
      "223 30 -0.0667781233788 0.000702759599916 0.00765780047483\n",
      "223 35 -0.0516590252519 0.000702759599916 0.00765740352326\n",
      "223 40 -0.0500149205327 0.000702759599916 0.00765704781392\n",
      "223 45 -0.0660746395588 0.000702759599916 0.00765664786981\n",
      "223 50 -0.0635121241212 0.000702759599916 0.00765631665163\n",
      "223 55 -0.0624483376741 0.000702759599916 0.00765600556726\n",
      "valid_acc 98.14666666666668\n",
      "224 0 -0.054538641125 0.000702759599916 0.00765553175996\n",
      "224 5 -0.0543782040477 0.000702759599916 0.00765510921305\n",
      "224 10 -0.0624535493553 0.000702759599916 0.00765459168921\n",
      "224 15 -0.0593404695392 0.000702759599916 0.00765384884196\n",
      "224 20 -0.0497372224927 0.000702759599916 0.00765299603746\n",
      "224 25 -0.049123570323 0.000702759599916 0.00765209070305\n",
      "224 30 -0.0574027784169 0.000702759599916 0.00765132824466\n",
      "224 35 -0.0503649897873 0.000702759599916 0.00765053465359\n",
      "224 40 -0.0449597984552 0.000702759599916 0.00764969545491\n",
      "224 45 -0.0405306741595 0.000702759599916 0.00764894512614\n",
      "224 50 -0.0379036031663 0.000702759599916 0.00764803511357\n",
      "224 55 -0.0646609514952 0.000702759599916 0.00764698616689\n",
      "valid_acc 98.16666666666667\n",
      "225 0 -0.0394821763039 0.000702759599916 0.00764604379095\n",
      "225 5 -0.0469438396394 0.000702759599916 0.00764508859571\n",
      "225 10 -0.0517460852861 0.000702759599916 0.00764419759682\n",
      "225 15 -0.0684076473117 0.000702759599916 0.0076433328971\n",
      "225 20 -0.0510530397296 0.000702759599916 0.00764213504015\n",
      "225 25 -0.0464412085712 0.000702759599916 0.0076407424647\n",
      "225 30 -0.0530396513641 0.000702759599916 0.00763935595269\n",
      "225 35 -0.0610517449677 0.000702759599916 0.0076379280083\n",
      "225 40 -0.0367381311953 0.000702759599916 0.00763666465983\n",
      "225 45 -0.0867851898074 0.000702759599916 0.00763541908245\n",
      "225 50 -0.0592862181365 0.000702759599916 0.00763403636345\n",
      "225 55 -0.0493957288563 0.000702759599916 0.00763276562442\n",
      "valid_acc 98.175\n",
      "226 0 -0.0731238424778 0.000702759599916 0.00763134713325\n",
      "226 5 -0.0624722577631 0.000702759599916 0.00762996090225\n",
      "226 10 -0.0606061480939 0.000702759599916 0.00762845628813\n",
      "226 15 -0.0482522808015 0.000702759599916 0.00762687703352\n",
      "226 20 -0.0435045547783 0.000702759599916 0.00762514986869\n",
      "226 25 -0.0534420609474 0.000702759599916 0.00762352100577\n",
      "226 30 -0.0552387610078 0.000702759599916 0.007621971592\n",
      "226 35 -0.0824435353279 0.000702759599916 0.00762070489323\n",
      "226 40 -0.0530783757567 0.000702759599916 0.00761941956149\n",
      "226 45 -0.0405416190624 0.000702759599916 0.00761802477387\n",
      "226 50 -0.0550328530371 0.000702759599916 0.00761663110802\n",
      "226 55 -0.0316858179867 0.000702759599916 0.00761514262015\n",
      "valid_acc 98.195\n",
      "227 0 -0.048152320087 0.000702759599916 0.00761371573517\n",
      "227 5 -0.0554903112352 0.000702759599916 0.0076125547871\n",
      "227 10 -0.0263869874179 0.000702759599916 0.00761147531103\n",
      "227 15 -0.0567524656653 0.000702759599916 0.00761063630144\n",
      "227 20 -0.0670138075948 0.000702759599916 0.00760986142817\n",
      "227 25 -0.0623711235821 0.000702759599916 0.00760891350874\n",
      "227 30 -0.0818332508206 0.000702759599916 0.00760803475554\n",
      "227 35 -0.0536760687828 0.000702759599916 0.00760714318311\n",
      "227 40 -0.0346780791879 0.000702759599916 0.00760624406502\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "227 45 -0.062593370676 0.000702759599916 0.00760526035733\n",
      "227 50 -0.0619517043233 0.000702759599916 0.00760453785689\n",
      "227 55 -0.0698467046022 0.000702759599916 0.00760402992847\n",
      "valid_acc 98.17833333333333\n",
      "228 0 -0.0738921687007 0.000702759599916 0.00760373153051\n",
      "228 5 -0.0703153535724 0.000702759599916 0.00760342954037\n",
      "228 10 -0.0667765513062 0.000702759599916 0.00760330810512\n",
      "228 15 -0.0706113427877 0.000702759599916 0.00760327357478\n",
      "228 20 -0.059385407716 0.000702759599916 0.00760313513905\n",
      "228 25 -0.0571623407304 0.000702759599916 0.00760288494116\n",
      "228 30 -0.0599103569984 0.000702759599916 0.00760254663645\n",
      "228 35 -0.0561603009701 0.000702759599916 0.00760237334501\n",
      "228 40 -0.051265090704 0.000702759599916 0.00760231340636\n",
      "228 45 -0.0734257921576 0.000702759599916 0.00760230328668\n",
      "228 50 -0.0392080657184 0.000702759599916 0.00760211382459\n",
      "228 55 -0.0555594936013 0.000702759599916 0.00760186505209\n",
      "valid_acc 98.13833333333334\n",
      "229 0 -0.0479273162782 0.000702759599916 0.00760171580199\n",
      "229 5 -0.0789807811379 0.000702759599916 0.00760157187799\n",
      "229 10 -0.0597353503108 0.000702759599916 0.00760137724398\n",
      "229 15 -0.0729920938611 0.000702759599916 0.00760130457886\n",
      "229 20 -0.0505069456995 0.000702759599916 0.00760134897569\n",
      "229 25 -0.087186768651 0.000702759599916 0.00760137658022\n",
      "229 30 -0.0422974862158 0.000702759599916 0.00760130064589\n",
      "229 35 -0.0651086717844 0.000702759599916 0.00760123730816\n",
      "229 40 -0.0403174832463 0.000702759599916 0.00760125922474\n",
      "229 45 -0.0410133451223 0.000702759599916 0.00760139539603\n",
      "229 50 -0.052372738719 0.000702759599916 0.00760141752198\n",
      "229 55 -0.0676972866058 0.000702759599916 0.00760109683704\n",
      "valid_acc 98.11999999999999\n",
      "230 0 -0.0548681728542 0.000702759599916 0.00760070398135\n",
      "230 5 -0.0480688102543 0.000702759599916 0.0076002272521\n",
      "230 10 -0.0499758608639 0.000702759599916 0.00759971695352\n",
      "230 15 -0.0622597970068 0.000702759599916 0.00759917840586\n",
      "230 20 -0.0691026672721 0.000702759599916 0.00759828017472\n",
      "230 25 -0.0509512536228 0.000702759599916 0.00759726673538\n",
      "230 30 -0.0480978749692 0.000702759599916 0.00759636053539\n",
      "230 35 -0.0450268350542 0.000702759599916 0.00759544159643\n",
      "230 40 -0.0476827472448 0.000702759599916 0.00759460238742\n",
      "230 45 -0.0472085960209 0.000702759599916 0.00759393769739\n",
      "230 50 -0.0376581922174 0.000702759599916 0.00759332406368\n",
      "230 55 -0.0402647331357 0.000702759599916 0.00759267972043\n",
      "valid_acc 98.11666666666666\n",
      "231 0 -0.0426692962646 0.000702759599916 0.0075919016304\n",
      "231 5 -0.0889100581408 0.000702759599916 0.0075911614667\n",
      "231 10 -0.0610068589449 0.000702759599916 0.00759047746736\n",
      "231 15 -0.0575486756861 0.000702759599916 0.00758971798979\n",
      "231 20 -0.0515135973692 0.000702759599916 0.00758885197204\n",
      "231 25 -0.0636562332511 0.000702759599916 0.00758809833865\n",
      "231 30 -0.0467826277018 0.000702759599916 0.00758752767561\n",
      "231 35 -0.0611911192536 0.000702759599916 0.00758688569822\n",
      "231 40 -0.0535270869732 0.000702759599916 0.00758623505698\n",
      "231 45 -0.0630986616015 0.000702759599916 0.00758566777947\n",
      "231 50 -0.0469493828714 0.000702759599916 0.00758506755122\n",
      "231 55 -0.0750188007951 0.000702759599916 0.00758456512527\n",
      "valid_acc 98.11\n",
      "232 0 -0.0637913942337 0.000702759599916 0.00758424886001\n",
      "232 5 -0.0361361242831 0.000702759599916 0.00758392548525\n",
      "232 10 -0.0408370643854 0.000702759599916 0.00758371486276\n",
      "232 15 -0.0478852838278 0.000702759599916 0.0075836899976\n",
      "232 20 -0.0714235976338 0.000702759599916 0.00758360389305\n",
      "232 25 -0.0484969504178 0.000702759599916 0.00758339899424\n",
      "232 30 -0.0624123513699 0.000702759599916 0.00758308079763\n",
      "232 35 -0.0890187621117 0.000702759599916 0.00758276221116\n",
      "232 40 -0.0640535354614 0.000702759599916 0.00758227212201\n",
      "232 45 -0.0395679399371 0.000702759599916 0.00758179165018\n",
      "232 50 -0.05051612854 0.000702759599916 0.00758100050343\n",
      "232 55 -0.0593102276325 0.000702759599916 0.00758017234431\n",
      "valid_acc 98.06833333333333\n",
      "233 0 -0.0488438531756 0.000702759599916 0.00757944410343\n",
      "233 5 -0.058419957757 0.000702759599916 0.00757875990687\n",
      "233 10 -0.055712684989 0.000702759599916 0.00757828257128\n",
      "233 15 -0.0528143011034 0.000702759599916 0.00757772601721\n",
      "233 20 -0.0641518086195 0.000702759599916 0.00757717800822\n",
      "233 25 -0.0475479662418 0.000702759599916 0.00757683520478\n",
      "233 30 -0.058868534863 0.000702759599916 0.00757651145266\n",
      "233 35 -0.0354011580348 0.000702759599916 0.00757631562972\n",
      "233 40 -0.0451380386949 0.000702759599916 0.00757623920776\n",
      "233 45 -0.0442046746612 0.000702759599916 0.00757610150531\n",
      "233 50 -0.0361346304417 0.000702759599916 0.00757570104497\n",
      "233 55 -0.0579235777259 0.000702759599916 0.00757515337182\n",
      "valid_acc 98.14166666666667\n",
      "234 0 -0.0661070793867 0.000702759599916 0.007574785532\n",
      "234 5 -0.0789922624826 0.000702759599916 0.00757432220518\n",
      "234 10 -0.0561430640519 0.000702759599916 0.00757372254177\n",
      "234 15 -0.0344642400742 0.000702759599916 0.00757335470644\n",
      "234 20 -0.0459957420826 0.000702759599916 0.00757298519046\n",
      "234 25 -0.0330815091729 0.000702759599916 0.00757252893271\n",
      "234 30 -0.0598799623549 0.000702759599916 0.00757199271592\n",
      "234 35 -0.0657606497407 0.000702759599916 0.00757128157604\n",
      "234 40 -0.0521797537804 0.000702759599916 0.00757039534229\n",
      "234 45 -0.0689410194755 0.000702759599916 0.00756957373759\n",
      "234 50 -0.0655209571123 0.000702759599916 0.00756855121276\n",
      "234 55 -0.0610189028084 0.000702759599916 0.00756741268998\n",
      "valid_acc 98.13166666666666\n",
      "235 0 -0.0703925415874 0.000702759599916 0.00756631266344\n",
      "235 5 -0.0730785429478 0.000702759599916 0.00756521665755\n",
      "235 10 -0.0370627902448 0.000702759599916 0.00756415969752\n",
      "235 15 -0.060265082866 0.000702759599916 0.00756308082322\n",
      "235 20 -0.0748466029763 0.000702759599916 0.00756189787961\n",
      "235 25 -0.0832670032978 0.000702759599916 0.00756087584803\n",
      "235 30 -0.0469631403685 0.000702759599916 0.00756007492971\n",
      "235 35 -0.0557839982212 0.000702759599916 0.00755946014284\n",
      "235 40 -0.0664790049195 0.000702759599916 0.0075588843354\n",
      "235 45 -0.0532567426562 0.000702759599916 0.00755843946601\n",
      "235 50 -0.0542978495359 0.000702759599916 0.00755814939263\n",
      "235 55 -0.0574426613748 0.000702759599916 0.00755788561963\n",
      "valid_acc 98.13333333333333\n",
      "236 0 -0.0447761118412 0.000702759599916 0.00755769909761\n",
      "236 5 -0.0556289553642 0.000702759599916 0.00755736243408\n",
      "236 10 -0.0570225268602 0.000702759599916 0.0075570414557\n",
      "236 15 -0.0471753068268 0.000702759599916 0.00755665855287\n",
      "236 20 -0.0543746128678 0.000702759599916 0.00755640895922\n",
      "236 25 -0.0445118248463 0.000702759599916 0.00755635817285\n",
      "236 30 -0.0418550372124 0.000702759599916 0.007556406567\n",
      "236 35 -0.0598960332572 0.000702759599916 0.00755632859852\n",
      "236 40 -0.0469429306686 0.000702759599916 0.00755614058714\n",
      "236 45 -0.0629458501935 0.000702759599916 0.00755591485226\n",
      "236 50 -0.0528036504984 0.000702759599916 0.00755557990374\n",
      "236 55 -0.0465073958039 0.000702759599916 0.00755523797321\n",
      "valid_acc 98.175\n",
      "237 0 -0.0568968169391 0.000702759599916 0.00755496392453\n",
      "237 5 -0.0416829623282 0.000702759599916 0.00755487490009\n",
      "237 10 -0.057284809649 0.000702759599916 0.00755479182037\n",
      "237 15 -0.0740807577968 0.000702759599916 0.00755451993584\n",
      "237 20 -0.0453009344637 0.000702759599916 0.00755408850003\n",
      "237 25 -0.0620626993477 0.000702759599916 0.00755356695687\n",
      "237 30 -0.0670446753502 0.000702759599916 0.00755297251854\n",
      "237 35 -0.0412457510829 0.000702759599916 0.00755214373732\n",
      "237 40 -0.0569912269711 0.000702759599916 0.00755128504792\n",
      "237 45 -0.0775843784213 0.000702759599916 0.00755058533918\n",
      "237 50 -0.0629732981324 0.000702759599916 0.00754974559296\n",
      "237 55 -0.0569002591074 0.000702759599916 0.00754877127169\n",
      "valid_acc 98.14166666666667\n",
      "238 0 -0.0534528195858 0.000702759599916 0.00754768090908\n",
      "238 5 -0.0556750893593 0.000702759599916 0.00754656621488\n",
      "238 10 -0.0562206730247 0.000702759599916 0.0075455262781\n",
      "238 15 -0.0442021414638 0.000702759599916 0.00754448242859\n",
      "238 20 -0.0992631986737 0.000702759599916 0.0075435396258\n",
      "238 25 -0.043295122683 0.000702759599916 0.00754266008156\n",
      "238 30 -0.0851669609547 0.000702759599916 0.00754186411951\n",
      "238 35 -0.0521097518504 0.000702759599916 0.0075410629049\n",
      "238 40 -0.0333608388901 0.000702759599916 0.00754020445326\n",
      "238 45 -0.0565216429532 0.000702759599916 0.00753920048697\n",
      "238 50 -0.0631531998515 0.000702759599916 0.007538132678\n",
      "238 55 -0.0474785752594 0.000702759599916 0.00753699882223\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_acc 98.13666666666667\n",
      "239 0 -0.0545619614422 0.000702759599916 0.0075358913156\n",
      "239 5 -0.0631695315242 0.000702759599916 0.00753470359268\n",
      "239 10 -0.0532643906772 0.000702759599916 0.00753363635299\n",
      "239 15 -0.0403149425983 0.000702759599916 0.00753269521582\n",
      "239 20 -0.0383047088981 0.000702759599916 0.00753170725062\n",
      "239 25 -0.0418851971626 0.000702759599916 0.007530707597\n",
      "239 30 -0.0641865804791 0.000702759599916 0.00752955791916\n",
      "239 35 -0.0527374483645 0.000702759599916 0.0075285650299\n",
      "239 40 -0.0394040644169 0.000702759599916 0.00752756943839\n",
      "239 45 -0.0738328546286 0.000702759599916 0.0075267315249\n",
      "239 50 -0.0571557767689 0.000702759599916 0.00752607070356\n",
      "239 55 -0.0337355583906 0.000702759599916 0.00752550856369\n",
      "valid_acc 98.18\n",
      "240 0 -0.0708226189017 0.000702759599916 0.007524865905\n",
      "240 5 -0.0602798089385 0.000702759599916 0.0075242137214\n",
      "240 10 -0.0703748092055 0.000702759599916 0.00752352446386\n",
      "240 15 -0.0407731719315 0.000702759599916 0.00752270865443\n",
      "240 20 -0.0729706957936 0.000702759599916 0.00752168627499\n",
      "240 25 -0.0445659570396 0.000702759599916 0.00752070985188\n",
      "240 30 -0.059781562537 0.000702759599916 0.00751985030574\n",
      "240 35 -0.0429108254611 0.000702759599916 0.00751898630652\n",
      "240 40 -0.0703123733401 0.000702759599916 0.00751801958845\n",
      "240 45 -0.0464846268296 0.000702759599916 0.00751707404705\n",
      "240 50 -0.0556655786932 0.000702759599916 0.00751616411178\n",
      "240 55 -0.068348094821 0.000702759599916 0.00751530653681\n",
      "valid_acc 98.15166666666667\n",
      "241 0 -0.0607740469277 0.000702759599916 0.00751452243594\n",
      "241 5 -0.069433182478 0.000702759599916 0.00751365835421\n",
      "241 10 -0.0408205613494 0.000702759599916 0.00751286477329\n",
      "241 15 -0.0325334556401 0.000702759599916 0.0075119903976\n",
      "241 20 -0.0528489425778 0.000702759599916 0.00751115962253\n",
      "241 25 -0.0914945453405 0.000702759599916 0.00751042353108\n",
      "241 30 -0.057069465518 0.000702759599916 0.00750966637865\n",
      "241 35 -0.0498356148601 0.000702759599916 0.00750905110604\n",
      "241 40 -0.0549615882337 0.000702759599916 0.00750845522175\n",
      "241 45 -0.0421884655952 0.000702759599916 0.00750789888243\n",
      "241 50 -0.0805269703269 0.000702759599916 0.00750727754037\n",
      "241 55 -0.0453771352768 0.000702759599916 0.00750668873314\n",
      "valid_acc 98.17333333333333\n",
      "242 0 -0.0358836166561 0.000702759599916 0.00750622172656\n",
      "242 5 -0.0410264357924 0.000702759599916 0.00750579508757\n",
      "242 10 -0.0634703710675 0.000702759599916 0.00750521878051\n",
      "242 15 -0.0565795302391 0.000702759599916 0.0075046474733\n",
      "242 20 -0.0457765050232 0.000702759599916 0.00750419424794\n",
      "242 25 -0.0663157626987 0.000702759599916 0.0075036863392\n",
      "242 30 -0.0615590065718 0.000702759599916 0.00750324091556\n",
      "242 35 -0.0491994358599 0.000702759599916 0.00750268235764\n",
      "242 40 -0.0795613825321 0.000702759599916 0.0075022286921\n",
      "242 45 -0.0407062880695 0.000702759599916 0.007501721524\n",
      "242 50 -0.0619869716465 0.000702759599916 0.00750130977721\n",
      "242 55 -0.0776444897056 0.000702759599916 0.00750107460032\n",
      "valid_acc 98.23166666666665\n",
      "best valid_acc 98.23166666666665\n",
      "243 0 -0.0579622536898 0.000702759599916 0.00750081004462\n",
      "243 5 -0.0445429868996 0.000702759599916 0.0075003348234\n",
      "243 10 -0.054954495281 0.000702759599916 0.0074998248506\n",
      "243 15 -0.0593411102891 0.000702759599916 0.00749904405973\n",
      "243 20 -0.0425831340253 0.000702759599916 0.00749807667343\n",
      "243 25 -0.0517974346876 0.000702759599916 0.00749711194378\n",
      "243 30 -0.0354294255376 0.000702759599916 0.007496155251\n",
      "243 35 -0.0854903012514 0.000702759599916 0.00749524445633\n",
      "243 40 -0.086198978126 0.000702759599916 0.00749422072359\n",
      "243 45 -0.0509626530111 0.000702759599916 0.00749304509762\n",
      "243 50 -0.0347561351955 0.000702759599916 0.00749185696182\n",
      "243 55 -0.0474366135895 0.000702759599916 0.0074906504736\n",
      "valid_acc 98.15833333333333\n",
      "244 0 -0.0571501925588 0.000702759599916 0.0074896212036\n",
      "244 5 -0.0396999120712 0.000702759599916 0.00748854853652\n",
      "244 10 -0.0423683859408 0.000702759599916 0.00748745865108\n",
      "244 15 -0.0591337420046 0.000702759599916 0.00748645550053\n",
      "244 20 -0.0718284919858 0.000702759599916 0.00748546091783\n",
      "244 25 -0.042693503201 0.000702759599916 0.00748452995691\n",
      "244 30 -0.056216109544 0.000702759599916 0.0074837951783\n",
      "244 35 -0.0495227202773 0.000702759599916 0.00748309325704\n",
      "244 40 -0.0321245677769 0.000702759599916 0.00748264902573\n",
      "244 45 -0.0525225996971 0.000702759599916 0.00748252033963\n",
      "244 50 -0.0438336469233 0.000702759599916 0.00748224490968\n",
      "244 55 -0.0502596870065 0.000702759599916 0.00748205419838\n",
      "valid_acc 98.205\n",
      "245 0 -0.0594159439206 0.000702759599916 0.00748210934362\n",
      "245 5 -0.0436013713479 0.000702759599916 0.00748214611628\n",
      "245 10 -0.0710546597838 0.000702759599916 0.00748219340353\n",
      "245 15 -0.0729502364993 0.000702759599916 0.00748204867161\n",
      "245 20 -0.0429270714521 0.000702759599916 0.00748187331434\n",
      "245 25 -0.055186778307 0.000702759599916 0.00748166664141\n",
      "245 30 -0.0334744155407 0.000702759599916 0.00748140950049\n",
      "245 35 -0.0352975912392 0.000702759599916 0.00748127011192\n",
      "245 40 -0.0645393952727 0.000702759599916 0.00748107445951\n",
      "245 45 -0.0653398036957 0.000702759599916 0.00748079997265\n",
      "245 50 -0.0396337658167 0.000702759599916 0.0074805519176\n",
      "245 55 -0.0479186028242 0.000702759599916 0.00748040358769\n",
      "valid_acc 98.10833333333333\n",
      "246 0 -0.0648098811507 0.000702759599916 0.0074802471714\n",
      "246 5 -0.0861347019672 0.000702759599916 0.00747996460248\n",
      "246 10 -0.0374947115779 0.000702759599916 0.00747958838153\n",
      "246 15 -0.0668787285686 0.000702759599916 0.00747939480321\n",
      "246 20 -0.0479185320437 0.000702759599916 0.00747909840176\n",
      "246 25 -0.0509259812534 0.000702759599916 0.0074789172617\n",
      "246 30 -0.061862424016 0.000702759599916 0.00747895787512\n",
      "246 35 -0.094880990684 0.000702759599916 0.0074788352101\n",
      "246 40 -0.0375670306385 0.000702759599916 0.0074784847402\n",
      "246 45 -0.0560249425471 0.000702759599916 0.00747794988355\n",
      "246 50 -0.0536122210324 0.000702759599916 0.00747726725134\n",
      "246 55 -0.0520915575325 0.000702759599916 0.00747644686994\n",
      "valid_acc 98.13\n",
      "247 0 -0.0564282312989 0.000702759599916 0.0074755173938\n",
      "247 5 -0.0470897778869 0.000702759599916 0.00747441012036\n",
      "247 10 -0.0481708906591 0.000702759599916 0.0074732891652\n",
      "247 15 -0.0562902018428 0.000702759599916 0.00747222943617\n",
      "247 20 -0.0719810724258 0.000702759599916 0.00747119634086\n",
      "247 25 -0.0451375134289 0.000702759599916 0.00747032395605\n",
      "247 30 -0.0511128529906 0.000702759599916 0.00746963610112\n",
      "247 35 -0.059052079916 0.000702759599916 0.00746882505104\n",
      "247 40 -0.0348122753203 0.000702759599916 0.00746802529701\n",
      "247 45 -0.0491698235273 0.000702759599916 0.00746733372885\n",
      "247 50 -0.0551143065095 0.000702759599916 0.00746676993033\n",
      "247 55 -0.043964702636 0.000702759599916 0.00746607942099\n",
      "valid_acc 98.14500000000001\n",
      "248 0 -0.0453109107912 0.000702759599916 0.00746527873085\n",
      "248 5 -0.0573832504451 0.000702759599916 0.00746468459118\n",
      "248 10 -0.0556596405804 0.000702759599916 0.00746422532\n",
      "248 15 -0.0631924495101 0.000702759599916 0.00746388576926\n",
      "248 20 -0.0701682567596 0.000702759599916 0.00746353372737\n",
      "248 25 -0.0589749924839 0.000702759599916 0.00746313522084\n",
      "248 30 -0.0611219257116 0.000702759599916 0.00746265540203\n",
      "248 35 -0.0422285273671 0.000702759599916 0.00746218330162\n",
      "248 40 -0.0794865787029 0.000702759599916 0.0074616481149\n",
      "248 45 -0.0688325688243 0.000702759599916 0.00746104262206\n",
      "248 50 -0.0636045187712 0.000702759599916 0.00746062785997\n",
      "248 55 -0.0680745691061 0.000702759599916 0.007460028779\n",
      "valid_acc 98.13666666666667\n",
      "249 0 -0.0663345456123 0.000702759599916 0.00745940484284\n",
      "249 5 -0.0747953355312 0.000702759599916 0.00745871455093\n",
      "249 10 -0.0682667195797 0.000702759599916 0.00745783060094\n",
      "249 15 -0.0788323581219 0.000702759599916 0.00745689717527\n",
      "249 20 -0.0552003160119 0.000702759599916 0.00745587009929\n",
      "249 25 -0.0418672934175 0.000702759599916 0.00745472056604\n",
      "249 30 -0.0509020611644 0.000702759599916 0.00745327591769\n",
      "249 35 -0.0737616345286 0.000702759599916 0.00745174964948\n",
      "249 40 -0.0244247112423 0.000702759599916 0.00745030954014\n",
      "249 45 -0.0253561977297 0.000702759599916 0.00744893579931\n",
      "249 50 -0.0532043911517 0.000702759599916 0.00744752613721\n",
      "249 55 -0.0758469253778 0.000702759599916 0.00744611234736\n",
      "valid_acc 98.165\n",
      "250 0 -0.0414002686739 0.000702759599916 0.00744462250648\n",
      "250 5 -0.0691305473447 0.000702759599916 0.00744325170971\n",
      "250 10 -0.0488805323839 0.000702759599916 0.00744206639513\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "250 15 -0.0443955846131 0.000702759599916 0.00744097188\n",
      "250 20 -0.0616896897554 0.000702759599916 0.00743971432803\n",
      "250 25 -0.0453078001738 0.000702759599916 0.00743861311731\n",
      "250 30 -0.0534292384982 0.000702759599916 0.00743753355879\n",
      "250 35 -0.0332022048533 0.000702759599916 0.00743649317686\n",
      "250 40 -0.0429191216826 0.000702759599916 0.00743560892889\n",
      "250 45 -0.0623906031251 0.000702759599916 0.00743477012104\n",
      "250 50 -0.0375207588077 0.000702759599916 0.00743404570031\n",
      "250 55 -0.0667971894145 0.000702759599916 0.00743341376367\n",
      "valid_acc 98.17833333333333\n",
      "251 0 -0.0373897962272 0.000702759599916 0.00743281660747\n",
      "251 5 -0.0425771400332 0.000702759599916 0.00743221981569\n",
      "251 10 -0.0418858714402 0.000702759599916 0.00743164910985\n",
      "251 15 -0.0760156065226 0.000702759599916 0.007431058496\n",
      "251 20 -0.0363539345562 0.000702759599916 0.00743062953064\n",
      "251 25 -0.049158398062 0.000702759599916 0.00743007824333\n",
      "251 30 -0.0387326292694 0.000702759599916 0.0074294968547\n",
      "251 35 -0.0578192695975 0.000702759599916 0.00742897045941\n",
      "251 40 -0.0439634919167 0.000702759599916 0.00742837804192\n",
      "251 45 -0.0463599935174 0.000702759599916 0.00742754390292\n",
      "251 50 -0.0549089759588 0.000702759599916 0.00742676729315\n",
      "251 55 -0.0418281033635 0.000702759599916 0.00742601477996\n",
      "valid_acc 98.18333333333334\n",
      "252 0 -0.0582223571837 -0.000415755860427 0.00742532294258\n",
      "252 5 -0.0352552123368 -0.000415755860427 0.0074244624793\n",
      "252 10 -0.0443168357015 -0.000415755860427 0.00742355762962\n",
      "252 15 -0.0597295090556 -0.000415755860427 0.00742262817304\n",
      "252 20 -0.0377067029476 -0.000415755860427 0.00742169255871\n",
      "252 25 -0.0775550380349 -0.000415755860427 0.00742092490734\n",
      "252 30 -0.0482814721763 -0.000415755860427 0.00742006015533\n",
      "252 35 -0.0768983885646 -0.000415755860427 0.00741916410554\n",
      "252 40 -0.0401491820812 -0.000415755860427 0.0074183916268\n",
      "252 45 -0.0506873950362 -0.000415755860427 0.00741768304551\n",
      "252 50 -0.0609411373734 -0.000415755860427 0.0074169778996\n",
      "252 55 -0.0313767828047 -0.000415755860427 0.00741619871077\n",
      "valid_acc 98.22166666666666\n",
      "253 0 -0.0510121807456 -0.000415755860427 0.00741528984393\n",
      "253 5 -0.0594900138676 -0.000415755860427 0.00741442801014\n",
      "253 10 -0.0532155260444 -0.000415755860427 0.00741359247952\n",
      "253 15 -0.0512528568506 -0.000415755860427 0.00741274010043\n",
      "253 20 -0.0542276129127 -0.000415755860427 0.00741203820722\n",
      "253 25 -0.0549205616117 -0.000415755860427 0.00741132377602\n",
      "253 30 -0.0674669146538 -0.000415755860427 0.00741062160845\n",
      "253 35 -0.097399763763 -0.000415755860427 0.00741003511129\n",
      "253 40 -0.0547022558749 -0.000415755860427 0.007409431244\n",
      "253 45 -0.0434817411005 -0.000415755860427 0.0074086099396\n",
      "253 50 -0.0334705486894 -0.000415755860427 0.00740758087378\n",
      "253 55 -0.0337662063539 -0.000415755860427 0.00740649200837\n",
      "valid_acc 98.17166666666667\n",
      "254 0 -0.0673875287175 -0.000415755860427 0.00740534173749\n",
      "254 5 -0.0715002566576 -0.000415755860427 0.0074040879024\n",
      "254 10 -0.0651588141918 -0.000415755860427 0.00740281327928\n",
      "254 15 -0.0443481691182 -0.000415755860427 0.00740150338153\n",
      "254 20 -0.0522045716643 -0.000415755860427 0.00740013858968\n",
      "254 25 -0.0510045439005 -0.000415755860427 0.00739864318558\n",
      "254 30 -0.0488410964608 -0.000415755860427 0.00739724556483\n",
      "254 35 -0.0701603665948 -0.000415755860427 0.00739589216162\n",
      "254 40 -0.0718292668462 -0.000415755860427 0.00739453905327\n",
      "254 45 -0.0889692381024 -0.000415755860427 0.00739313491379\n",
      "254 50 -0.0515339449048 -0.000415755860427 0.00739165811579\n",
      "254 55 -0.0872232019901 -0.000415755860427 0.0073901954329\n",
      "valid_acc 98.185\n",
      "255 0 -0.0633075609803 -0.000415755860427 0.00738880942203\n",
      "255 5 -0.0646075680852 -0.000415755860427 0.00738738030581\n",
      "255 10 -0.0618283525109 -0.000415755860427 0.00738586683415\n",
      "255 15 -0.0531779043376 -0.000415755860427 0.00738435293523\n",
      "255 20 -0.0607592724264 -0.000415755860427 0.00738310970881\n",
      "255 25 -0.0623244233429 -0.000415755860427 0.0073821376605\n",
      "255 30 -0.0949973836541 -0.000415755860427 0.00738122527153\n",
      "255 35 -0.0672455355525 -0.000415755860427 0.00738037315064\n",
      "255 40 -0.0685821250081 -0.000415755860427 0.00737940119273\n",
      "255 45 -0.0367623828351 -0.000415755860427 0.00737820375129\n",
      "255 50 -0.0477485358715 -0.000415755860427 0.00737706586472\n",
      "255 55 -0.0500174462795 -0.000415755860427 0.0073758487707\n",
      "valid_acc 98.22833333333332\n",
      "256 0 -0.0617053732276 -0.000415755860427 0.00737443885743\n",
      "256 5 -0.0558934770525 -0.000415755860427 0.00737312920178\n",
      "256 10 -0.0510845594108 -0.000415755860427 0.00737190546819\n",
      "256 15 -0.0584193244576 -0.000415755860427 0.00737068575685\n",
      "256 20 -0.0669648423791 -0.000415755860427 0.0073695569526\n",
      "256 25 -0.0525647737086 -0.000415755860427 0.00736853524069\n",
      "256 30 -0.0664140731096 -0.000415755860427 0.00736757767327\n",
      "256 35 -0.0410250015557 -0.000415755860427 0.00736679818363\n",
      "256 40 -0.0738963782787 -0.000415755860427 0.00736602465115\n",
      "256 45 -0.0477630011737 -0.000415755860427 0.00736543449593\n",
      "256 50 -0.056305129081 -0.000415755860427 0.00736495561643\n",
      "256 55 -0.047327503562 -0.000415755860427 0.00736430428714\n",
      "valid_acc 98.16666666666667\n",
      "257 0 -0.0484672077 -0.000415755860427 0.00736369802796\n",
      "257 5 -0.0337959639728 -0.000415755860427 0.00736309656848\n",
      "257 10 -0.0469299852848 -0.000415755860427 0.00736230381816\n",
      "257 15 -0.0398737937212 -0.000415755860427 0.00736131312026\n",
      "257 20 -0.050309445709 -0.000415755860427 0.00736022127805\n",
      "257 25 -0.0843009799719 -0.000415755860427 0.00735920474\n",
      "257 30 -0.0487974919379 -0.000415755860427 0.00735832578724\n",
      "257 35 -0.049206674099 -0.000415755860427 0.00735751692646\n",
      "257 40 -0.0636084005237 -0.000415755860427 0.00735648147532\n",
      "257 45 -0.0979380235076 -0.000415755860427 0.00735545195201\n",
      "257 50 -0.0754362866282 -0.000415755860427 0.00735434952478\n",
      "257 55 -0.0533895902336 -0.000415755860427 0.00735330915867\n",
      "valid_acc 98.20166666666667\n",
      "258 0 -0.0607846304774 -0.000415755860427 0.00735232631087\n",
      "258 5 -0.0503409318626 -0.000415755860427 0.00735142204917\n",
      "258 10 -0.0422723777592 -0.000415755860427 0.00735053295312\n",
      "258 15 -0.0628523156047 -0.000415755860427 0.00734972833986\n",
      "258 20 -0.0592557564378 -0.000415755860427 0.00734873188573\n",
      "258 25 -0.053338073194 -0.000415755860427 0.00734759308071\n",
      "258 30 -0.0348901003599 -0.000415755860427 0.00734646840211\n",
      "258 35 -0.0662618279457 -0.000415755860427 0.00734522966853\n",
      "258 40 -0.0518678128719 -0.000415755860427 0.00734410953837\n",
      "258 45 -0.0501131154597 -0.000415755860427 0.00734299924515\n",
      "258 50 -0.0491159558296 -0.000415755860427 0.00734200663215\n",
      "258 55 -0.0571557953954 -0.000415755860427 0.00734079265821\n",
      "valid_acc 98.21666666666667\n",
      "259 0 -0.0370741784573 -0.000415755860427 0.00733956105564\n",
      "259 5 -0.0521148405969 -0.000415755860427 0.00733816238601\n",
      "259 10 -0.0494885481894 -0.000415755860427 0.00733665228001\n",
      "259 15 -0.0515739656985 -0.000415755860427 0.00733505544259\n",
      "259 20 -0.0491133332253 -0.000415755860427 0.00733337885719\n",
      "259 25 -0.0522849038243 -0.000415755860427 0.00733170089282\n",
      "259 30 -0.0345835462213 -0.000415755860427 0.00733001756125\n",
      "259 35 -0.0676570534706 -0.000415755860427 0.0073284553817\n",
      "259 40 -0.0519363768399 -0.000415755860427 0.00732695200823\n",
      "259 45 -0.0692522451282 -0.000415755860427 0.00732548599594\n",
      "259 50 -0.0489766336977 -0.000415755860427 0.00732406530635\n",
      "259 55 -0.0780707225204 -0.000415755860427 0.00732260881638\n",
      "valid_acc 98.26333333333334\n",
      "best valid_acc 98.26333333333334\n",
      "260 0 -0.0611028037965 -0.000415755860427 0.00732114793647\n",
      "260 5 -0.0578362457454 -0.000415755860427 0.00731967052521\n",
      "260 10 -0.0601168684661 -0.000415755860427 0.00731804543355\n",
      "260 15 -0.061509475112 -0.000415755860427 0.00731617483688\n",
      "260 20 -0.0504107549787 -0.000415755860427 0.00731427403928\n",
      "260 25 -0.0396412648261 -0.000415755860427 0.00731252688173\n",
      "260 30 -0.0587942451239 -0.000415755860427 0.00731084022964\n",
      "260 35 -0.0545634292066 -0.000415755860427 0.00730925908118\n",
      "260 40 -0.0507907904685 -0.000415755860427 0.00730780226258\n",
      "260 45 -0.0460444353521 -0.000415755860427 0.00730661547421\n",
      "260 50 -0.0674875825644 -0.000415755860427 0.00730539033352\n",
      "260 55 -0.0522698201239 -0.000415755860427 0.00730406890476\n",
      "valid_acc 98.16666666666667\n",
      "261 0 -0.054542068392 -0.000415755860427 0.00730277926067\n",
      "261 5 -0.054691310972 -0.000415755860427 0.00730149456592\n",
      "261 10 -0.0434978939593 -0.000415755860427 0.00730009324167\n",
      "261 15 -0.0350395515561 -0.000415755860427 0.00729877450877\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "261 20 -0.0609051771462 -0.000415755860427 0.00729751822919\n",
      "261 25 -0.0604960247874 -0.000415755860427 0.00729631926706\n",
      "261 30 -0.0752865597606 -0.000415755860427 0.00729505000897\n",
      "261 35 -0.0289634875953 -0.000415755860427 0.00729374897028\n",
      "261 40 -0.0665903016925 -0.000415755860427 0.00729246001455\n",
      "261 45 -0.0307307895273 -0.000415755860427 0.00729111297232\n",
      "261 50 -0.0524453818798 -0.000415755860427 0.00728972984965\n",
      "261 55 -0.0717144683003 -0.000415755860427 0.0072883707883\n",
      "valid_acc 98.18666666666667\n",
      "262 0 -0.0578728094697 -0.000415755860427 0.00728697211169\n",
      "262 5 -0.0836705118418 -0.000415755860427 0.00728567000447\n",
      "262 10 -0.0875814259052 -0.000415755860427 0.00728470098967\n",
      "262 15 -0.042802054435 -0.000415755860427 0.00728385897529\n",
      "262 20 -0.049227103591 -0.000415755860427 0.00728291459238\n",
      "262 25 -0.06529250741 -0.000415755860427 0.00728190715413\n",
      "262 30 -0.0508093200624 -0.000415755860427 0.00728079290938\n",
      "262 35 -0.0394474864006 -0.000415755860427 0.0072794667572\n",
      "262 40 -0.0515846759081 -0.000415755860427 0.00727803470005\n",
      "262 45 -0.0757287666202 -0.000415755860427 0.00727636317732\n",
      "262 50 -0.0445034094155 -0.000415755860427 0.00727451460825\n",
      "262 55 -0.0759842693806 -0.000415755860427 0.0072725339854\n",
      "valid_acc 98.16833333333334\n",
      "263 0 -0.0584321022034 -0.000415755860427 0.00727047295976\n",
      "263 5 -0.034319255501 -0.000415755860427 0.00726852156502\n",
      "263 10 -0.0560624711215 -0.000415755860427 0.00726646059517\n",
      "263 15 -0.0502281039953 -0.000415755860427 0.00726426806717\n",
      "263 20 -0.0644133239985 -0.000415755860427 0.00726194931868\n",
      "263 25 -0.0461356192827 -0.000415755860427 0.00725966869131\n",
      "263 30 -0.0425854846835 -0.000415755860427 0.00725731891534\n",
      "263 35 -0.0483965873718 -0.000415755860427 0.00725488299454\n",
      "263 40 -0.0384861528873 -0.000415755860427 0.00725233136133\n",
      "263 45 -0.0600221008062 -0.000415755860427 0.00724988350214\n",
      "263 50 -0.0593372024596 -0.000415755860427 0.00724732923717\n",
      "263 55 -0.0457340441644 -0.000415755860427 0.00724468715301\n",
      "valid_acc 98.18333333333334\n",
      "264 0 -0.0520397163928 -0.000415755860427 0.00724204505854\n",
      "264 5 -0.0600007586181 -0.000415755860427 0.0072393568928\n",
      "264 10 -0.0382459573448 -0.000415755860427 0.00723672207647\n",
      "264 15 -0.0526390522718 -0.000415755860427 0.00723424022015\n",
      "264 20 -0.039838463068 -0.000415755860427 0.0072319078235\n",
      "264 25 -0.0539014786482 -0.000415755860427 0.00722994319963\n",
      "264 30 -0.102971538901 -0.000415755860427 0.00722786523016\n",
      "264 35 -0.0455195605755 -0.000415755860427 0.0072258748819\n",
      "264 40 -0.0442675240338 -0.000415755860427 0.00722391520896\n",
      "264 45 -0.0634308606386 -0.000415755860427 0.00722192653582\n",
      "264 50 -0.0402177795768 -0.000415755860427 0.00721983393104\n",
      "264 55 -0.0551637746394 -0.000415755860427 0.00721782996212\n",
      "valid_acc 98.15\n",
      "265 0 -0.0675570517778 -0.000415755860427 0.00721611071005\n",
      "265 5 -0.0631123259664 -0.000415755860427 0.00721456335321\n",
      "265 10 -0.0574302859604 -0.000415755860427 0.00721312318843\n",
      "265 15 -0.045920509845 -0.000415755860427 0.00721175163406\n",
      "265 20 -0.07574788481 -0.000415755860427 0.00721049562916\n",
      "265 25 -0.0648438036442 -0.000415755860427 0.00720928356777\n",
      "265 30 -0.0519146323204 -0.000415755860427 0.00720820669406\n",
      "265 35 -0.0677389577031 -0.000415755860427 0.00720719323537\n",
      "265 40 -0.0592929907143 -0.000415755860427 0.0072060286172\n",
      "265 45 -0.0533041991293 -0.000415755860427 0.00720478710764\n",
      "265 50 -0.0483317263424 -0.000415755860427 0.00720342914921\n",
      "265 55 -0.0554487779737 -0.000415755860427 0.0072021174192\n",
      "valid_acc 98.19\n",
      "266 0 -0.047525729984 -0.000415755860427 0.00720082022999\n",
      "266 5 -0.0521975718439 -0.000415755860427 0.00719961774436\n",
      "266 10 -0.0490166135132 -0.000415755860427 0.00719840400163\n",
      "266 15 -0.0600587762892 -0.000415755860427 0.00719721679291\n",
      "266 20 -0.0539117902517 -0.000415755860427 0.00719627609369\n",
      "266 25 -0.0740914568305 -0.000415755860427 0.00719540211865\n",
      "266 30 -0.0570370480418 -0.000415755860427 0.00719446036272\n",
      "266 35 -0.0688514411449 -0.000415755860427 0.00719340726908\n",
      "266 40 -0.0483980365098 -0.000415755860427 0.00719214693946\n",
      "266 45 -0.0909596532583 -0.000415755860427 0.00719074375599\n",
      "266 50 -0.0421287082136 -0.000415755860427 0.00718940257501\n",
      "266 55 -0.0744553431869 -0.000415755860427 0.00718805416921\n",
      "valid_acc 98.205\n",
      "267 0 -0.0698519125581 -0.000415755860427 0.0071866719119\n",
      "267 5 -0.0756399780512 -0.000415755860427 0.00718526107998\n",
      "267 10 -0.0445787012577 -0.000415755860427 0.0071838336366\n",
      "267 15 -0.0708724930882 -0.000415755860427 0.00718241895059\n",
      "267 20 -0.0486487597227 -0.000415755860427 0.00718106599426\n",
      "267 25 -0.0594642423093 -0.000415755860427 0.00717975241111\n",
      "267 30 -0.0500227063894 -0.000415755860427 0.00717849647922\n",
      "267 35 -0.0527340397239 -0.000415755860427 0.00717734995882\n",
      "267 40 -0.0720193609595 -0.000415755860427 0.00717610307094\n",
      "267 45 -0.0663803145289 -0.000415755860427 0.00717492889801\n",
      "267 50 -0.0610560923815 -0.000415755860427 0.00717407463589\n",
      "267 55 -0.0794172137976 -0.000415755860427 0.00717317307972\n",
      "valid_acc 98.24666666666667\n",
      "268 0 -0.0658733993769 -0.000415755860427 0.00717214034843\n",
      "268 5 -0.0664323270321 -0.000415755860427 0.00717118202584\n",
      "268 10 -0.0495977550745 -0.000258310997682 0.0071701954179\n",
      "268 15 -0.0619206018746 -0.000258310997682 0.00716916030777\n",
      "268 20 -0.0372456982732 -0.000258310997682 0.00716806099111\n",
      "268 25 -0.0474489070475 -0.000258310997682 0.00716697094054\n",
      "268 30 -0.0619868561625 -0.000258310997682 0.00716607695454\n",
      "268 35 -0.050049521029 -0.000258310997682 0.00716498619195\n",
      "268 40 -0.0477584712207 -0.000258310997682 0.00716375629553\n",
      "268 45 -0.0776041224599 -0.000258310997682 0.00716253732943\n",
      "268 50 -0.0658115670085 -0.000258310997682 0.00716129394377\n",
      "268 55 -0.0534950532019 -0.000258310997682 0.00716005680056\n",
      "valid_acc 98.29833333333333\n",
      "best valid_acc 98.29833333333333\n",
      "269 0 -0.0505832508206 -0.000258310997682 0.00715885251117\n",
      "269 5 -0.0483590848744 -0.000258310997682 0.00715761967264\n",
      "269 10 -0.0619766078889 -0.000258310997682 0.00715641678069\n",
      "269 15 -0.0602109767497 -0.000258310997682 0.0071551406886\n",
      "269 20 -0.0300504397601 -0.000258310997682 0.00715392371626\n",
      "269 25 -0.0486758574843 -0.000258310997682 0.00715266964691\n",
      "269 30 -0.0708196908236 -0.000258310997682 0.00715127148084\n",
      "269 35 -0.0427494719625 -0.000258310997682 0.00714971453589\n",
      "269 40 -0.0463726632297 -0.000258310997682 0.00714799314341\n",
      "269 45 -0.0598600581288 -0.000258310997682 0.00714619403334\n",
      "269 50 -0.0797225385904 -0.000258310997682 0.00714429695937\n",
      "269 55 -0.0452710986137 -0.000258310997682 0.00714229627859\n",
      "valid_acc 98.24000000000001\n",
      "270 0 -0.0623299628496 -0.000258310997682 0.00714047789689\n",
      "270 5 -0.0432651974261 -0.000258310997682 0.00713880265774\n",
      "270 10 -0.0250092390925 -0.000258310997682 0.00713703057732\n",
      "270 15 -0.0569441653788 -0.000258310997682 0.00713520896749\n",
      "270 20 -0.0333368219435 -0.000258310997682 0.00713339424356\n",
      "270 25 -0.0625647976995 -0.000258310997682 0.00713167205016\n",
      "270 30 -0.0543592497706 -0.000258310997682 0.00713021341017\n",
      "270 35 -0.0440548956394 -0.000258310997682 0.00712880308981\n",
      "270 40 -0.123391963542 -0.000258310997682 0.00712734568908\n",
      "270 45 -0.0549581684172 -0.000258310997682 0.00712587328526\n",
      "270 50 -0.0524405911565 -0.000258310997682 0.00712463407511\n",
      "270 55 -0.045423656702 -0.000258310997682 0.00712326807423\n",
      "valid_acc 98.27\n",
      "271 0 -0.0534315519035 -0.000258310997682 0.00712199569735\n",
      "271 5 -0.0404119715095 -0.000258310997682 0.00712068146831\n",
      "271 10 -0.0840374827385 -0.000258310997682 0.00711924956962\n",
      "271 15 -0.0561097040772 -0.000258310997682 0.00711791335602\n",
      "271 20 -0.0669894069433 -0.000258310997682 0.00711664351965\n",
      "271 25 -0.0399260818958 -0.000258310997682 0.0071152858712\n",
      "271 30 -0.050484161824 -0.000258310997682 0.00711400775012\n",
      "271 35 -0.0571189038455 -0.000258310997682 0.00711289939789\n",
      "271 40 -0.0644691586494 -0.000258310997682 0.00711193276865\n",
      "271 45 -0.0518275052309 -0.000258310997682 0.00711115303646\n",
      "271 50 -0.0514771342278 -0.000258310997682 0.00711046856768\n",
      "271 55 -0.0526162870228 -0.000258310997682 0.0071098245804\n",
      "valid_acc 98.25666666666667\n",
      "272 0 -0.0569291748106 -0.000258310997682 0.00710909256072\n",
      "272 5 -0.0586320236325 -0.000258310997682 0.00710840968867\n",
      "272 10 -0.0504731237888 -0.000258310997682 0.0071077586092\n",
      "272 15 -0.0663747116923 -0.000258310997682 0.00710731538464\n",
      "272 20 -0.0543768629432 -0.000258310997682 0.00710677542005\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "272 25 -0.0342307277024 -0.000258310997682 0.00710605514558\n",
      "272 30 -0.0470339879394 -0.000258310997682 0.00710522974121\n",
      "272 35 -0.0305077396333 -0.000258310997682 0.00710449283802\n",
      "272 40 -0.0458019413054 -0.000258310997682 0.00710399522019\n",
      "272 45 -0.0396391153336 -0.000258310997682 0.00710363648385\n",
      "272 50 -0.0622355379164 -0.000258310997682 0.00710338898687\n",
      "272 55 -0.0460151731968 -0.000258310997682 0.00710316137313\n",
      "valid_acc 98.27166666666668\n",
      "273 0 -0.0467827096581 -0.000258310997682 0.00710292768098\n",
      "273 5 -0.057935308665 -0.000258310997682 0.00710262883418\n",
      "273 10 -0.0574551932514 -0.000258310997682 0.00710233708211\n",
      "273 15 -0.0627723708749 -0.000258310997682 0.00710239113593\n",
      "273 20 -0.0791785940528 -0.000258310997682 0.00710250047376\n",
      "273 25 -0.0353898927569 -0.000258310997682 0.00710261242162\n",
      "273 30 -0.0725312307477 -0.000258310997682 0.00710268420532\n",
      "273 35 -0.048637278378 -0.000258310997682 0.00710282226133\n",
      "273 40 -0.0560430511832 -0.000258310997682 0.00710297368849\n",
      "273 45 -0.0544077083468 -0.000258310997682 0.00710287775051\n",
      "273 50 -0.0805306881666 -0.000258310997682 0.00710280224372\n",
      "273 55 -0.0504662171006 -0.000258310997682 0.00710268640091\n",
      "valid_acc 98.23666666666668\n",
      "274 0 -0.0595253631473 -0.000258310997682 0.00710254585773\n",
      "274 5 -0.063309662044 -0.000258310997682 0.00710227970237\n",
      "274 10 -0.0467909686267 -0.000258310997682 0.00710189150655\n",
      "274 15 -0.0698831230402 -0.000258310997682 0.00710129203181\n",
      "274 20 -0.051630191505 -0.000258310997682 0.00710067195897\n",
      "274 25 -0.0641557276249 -0.000258310997682 0.00710012952766\n",
      "274 30 -0.0668143928051 -0.000258310997682 0.00709977218985\n",
      "274 35 -0.0513221360743 -0.000258310997682 0.00709939067563\n",
      "274 40 -0.049443077296 -0.000258310997682 0.00709879088224\n",
      "274 45 -0.04533784464 -0.000258310997682 0.00709837988319\n",
      "274 50 -0.0409172102809 -0.000258310997682 0.0070981777517\n",
      "274 55 -0.0495326109231 -0.000258310997682 0.00709804599394\n",
      "valid_acc 98.24833333333333\n",
      "275 0 -0.0499217994511 -0.000258310997682 0.00709809213585\n",
      "275 5 -0.0298766046762 -0.000258310997682 0.00709818991674\n",
      "275 10 -0.0690325945616 -0.000258310997682 0.00709829579787\n",
      "275 15 -0.0527882874012 -0.000258310997682 0.00709823531075\n",
      "275 20 -0.055985622108 -0.000258310997682 0.00709808198078\n",
      "275 25 -0.0683798119426 -0.000258310997682 0.00709800696854\n",
      "275 30 -0.0531286858022 -0.000258310997682 0.0070979243099\n",
      "275 35 -0.0531361922622 -0.000258310997682 0.00709777357505\n",
      "275 40 -0.0604366064072 -0.000258310997682 0.00709766136056\n",
      "275 45 -0.0577796325088 -0.000258310997682 0.00709742754128\n",
      "275 50 -0.0518176406622 -0.000258310997682 0.00709712691532\n",
      "275 55 -0.0697737559676 -0.000258310997682 0.00709673254381\n",
      "valid_acc 98.27666666666667\n",
      "276 0 -0.0628878995776 -0.000258310997682 0.00709628873641\n",
      "276 5 -0.0484262444079 -0.000258310997682 0.00709586276404\n",
      "276 10 -0.0559813342988 -0.000258310997682 0.00709566032929\n",
      "276 15 -0.0681895390153 -0.000258310997682 0.00709551076244\n",
      "276 20 -0.0514152422547 -0.000258310997682 0.00709530062412\n",
      "276 25 -0.0606863126159 -0.000258310997682 0.00709510256376\n",
      "276 30 -0.0523803420365 -0.000258310997682 0.00709493823839\n",
      "276 35 -0.0433219596744 -0.000258310997682 0.00709468986209\n",
      "276 40 -0.0323233790696 -0.000258310997682 0.00709455884294\n",
      "276 45 -0.0282671097666 -0.000258310997682 0.00709421852432\n",
      "276 50 -0.0447864718735 -0.000258310997682 0.00709387239884\n",
      "276 55 -0.0494671799242 -0.000258310997682 0.00709354438819\n",
      "valid_acc 98.27\n",
      "277 0 -0.0573457106948 -0.000258310997682 0.0070931164859\n",
      "277 5 -0.0401163287461 -0.000258310997682 0.00709256498835\n",
      "277 10 -0.0625213235617 -0.000258310997682 0.00709186937442\n",
      "277 15 -0.0698091611266 -0.000258310997682 0.00709125290934\n",
      "277 20 -0.069587007165 -0.000258310997682 0.00709048632863\n",
      "277 25 -0.0446111671627 -0.000258310997682 0.00708962689444\n",
      "277 30 -0.0495184995234 -0.000258310997682 0.00708882614955\n",
      "277 35 -0.03951979056 -0.000258310997682 0.00708801077497\n",
      "277 40 -0.0356934666634 -0.000258310997682 0.0070870693628\n",
      "277 45 -0.0614431612194 -0.000258310997682 0.00708605614984\n",
      "277 50 -0.0669827535748 -0.000258310997682 0.00708495228704\n",
      "277 55 -0.0407481379807 -0.000258310997682 0.00708390162963\n",
      "valid_acc 98.36166666666666\n",
      "best valid_acc 98.36166666666666\n",
      "278 0 -0.0314596705139 -0.000258310997682 0.0070828716782\n",
      "278 5 -0.0735886320472 -0.000258310997682 0.00708178023024\n",
      "278 10 -0.0466703884304 -0.000258310997682 0.00708045753768\n",
      "278 15 -0.0312806218863 -0.000258310997682 0.00707899917264\n",
      "278 20 -0.0423500798643 -0.000258310997682 0.00707747946489\n",
      "278 25 -0.033084012568 -0.000258310997682 0.00707601587462\n",
      "278 30 -0.086002163589 -0.000258310997682 0.0070744578737\n",
      "278 35 -0.0376441702247 -0.000258310997682 0.00707285226047\n",
      "278 40 -0.0294639971107 -0.000258310997682 0.00707098828032\n",
      "278 45 -0.0588301420212 -0.000258310997682 0.00706906624546\n",
      "278 50 -0.0325069800019 -0.000258310997682 0.00706708769093\n",
      "278 55 -0.0552532635629 -0.000258310997682 0.00706509721107\n",
      "valid_acc 98.355\n",
      "279 0 -0.0553198605776 -0.000258310997682 0.00706301831908\n",
      "279 5 -0.0501385852695 -0.000258310997682 0.00706097045569\n",
      "279 10 -0.0290678068995 -0.000258310997682 0.00705902310843\n",
      "279 15 -0.0589609257877 -0.000258310997682 0.00705719294194\n",
      "279 20 -0.0700824409723 -0.000258310997682 0.00705532472202\n",
      "279 25 -0.035902030766 -0.000258310997682 0.00705349553693\n",
      "279 30 -0.0586323142052 -0.000258310997682 0.00705155594598\n",
      "279 35 -0.0231544487178 -0.000258310997682 0.00704965323897\n",
      "279 40 -0.0702584385872 -0.000258310997682 0.00704787463318\n",
      "279 45 -0.0382833667099 -0.000258310997682 0.00704612838415\n",
      "279 50 -0.0868238285184 -0.000258310997682 0.00704457671142\n",
      "279 55 -0.0443212203681 -0.000258310997682 0.00704306152866\n",
      "valid_acc 98.34333333333333\n",
      "280 0 -0.0363973155618 -0.000258310997682 0.00704146913938\n",
      "280 5 -0.0566831454635 -0.000258310997682 0.00703989881345\n",
      "280 10 -0.054923992604 -0.000258310997682 0.00703843635687\n",
      "280 15 -0.0727866962552 -0.000258310997682 0.00703694890819\n",
      "280 20 -0.0478872545063 -0.000258310997682 0.00703535777978\n",
      "280 25 -0.0656313225627 -0.000258310997682 0.00703379897124\n",
      "280 30 -0.0464093238115 -0.000258310997682 0.00703229271596\n",
      "280 35 -0.0736170560122 -0.000258310997682 0.00703097425724\n",
      "280 40 -0.0596112124622 -0.000258310997682 0.00702987486052\n",
      "280 45 -0.0443401932716 -0.000258310997682 0.00702876114018\n",
      "280 50 -0.0715424418449 -0.000258310997682 0.00702778311453\n",
      "280 55 -0.02982047759 -0.000258310997682 0.00702690770633\n",
      "valid_acc 98.32666666666667\n",
      "281 0 -0.0448926240206 -0.000258310997682 0.00702613501142\n",
      "281 5 -0.061932310462 -0.000258310997682 0.00702536725983\n",
      "281 10 -0.0402091555297 -0.000258310997682 0.00702432646485\n",
      "281 15 -0.0891790091991 -0.000258310997682 0.00702334533467\n",
      "281 20 -0.0580991357565 -0.000258310997682 0.00702224179031\n",
      "281 25 -0.0512809529901 -0.000258310997682 0.00702104911167\n",
      "281 30 -0.0431775189936 -0.000258310997682 0.00701981650351\n",
      "281 35 -0.048326831311 -0.000258310997682 0.00701875092546\n",
      "281 40 -0.0403513647616 -0.000258310997682 0.00701776112877\n",
      "281 45 -0.058994948864 -0.000258310997682 0.00701668506304\n",
      "281 50 -0.0551964491606 -0.000258310997682 0.00701580502672\n",
      "281 55 -0.0421861894429 -0.000258310997682 0.00701488641207\n",
      "valid_acc 98.31333333333333\n",
      "282 0 -0.0449666976929 -0.000258310997682 0.00701380595911\n",
      "282 5 -0.0478696115315 -0.000258310997682 0.0070128194606\n",
      "282 10 -0.0369209237397 -0.000258310997682 0.00701186980714\n",
      "282 15 -0.0625330135226 -0.000258310997682 0.00701093317974\n",
      "282 20 -0.0432990789413 -0.000258310997682 0.00700985795121\n",
      "282 25 -0.059138212353 -0.000258310997682 0.00700869485876\n",
      "282 30 -0.0622971393168 -0.000258310997682 0.00700749856344\n",
      "282 35 -0.0571938492358 -0.000258310997682 0.00700635242823\n",
      "282 40 -0.0575817003846 -0.000258310997682 0.00700511231361\n",
      "282 45 -0.0363710783422 -0.000258310997682 0.00700389818756\n",
      "282 50 -0.0490932352841 -0.000258310997682 0.00700281311939\n",
      "282 55 -0.0400292724371 -0.000258310997682 0.00700156215894\n",
      "valid_acc 98.315\n",
      "283 0 -0.0443548969924 -0.000258310997682 0.00700041081726\n",
      "283 5 -0.0348140969872 -0.000258310997682 0.00699930613641\n",
      "283 10 -0.0540993399918 -0.000258310997682 0.00699837873419\n",
      "283 15 -0.0561519563198 -0.000258310997682 0.00699762860736\n",
      "283 20 -0.0443399846554 -0.000258310997682 0.00699679297875\n",
      "283 25 -0.0462501235306 -0.000258310997682 0.0069958344546\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "283 30 -0.0328531526029 -0.000258310997682 0.00699483214588\n",
      "283 35 -0.052335575223 -0.000258310997682 0.00699376053096\n",
      "283 40 -0.0705074965954 -0.000258310997682 0.00699259470706\n",
      "283 45 -0.0678800269961 -0.000258310997682 0.00699144062668\n",
      "283 50 -0.0450191907585 -0.000258310997682 0.00699034459393\n",
      "283 55 -0.0630781948566 -0.000258310997682 0.00698941298684\n",
      "valid_acc 98.28833333333333\n",
      "284 0 -0.0413851290941 -0.000258310997682 0.00698866311043\n",
      "284 5 -0.0319217666984 -0.000258310997682 0.00698794420654\n",
      "284 10 -0.079034075141 -0.000258310997682 0.00698723070532\n",
      "284 15 -0.0565723031759 -0.000258310997682 0.00698639157181\n",
      "284 20 -0.0524921827018 -0.000258310997682 0.00698558051088\n",
      "284 25 -0.0503051429987 -0.000258310997682 0.00698498913072\n",
      "284 30 -0.0486239306629 -0.000258310997682 0.00698426521242\n",
      "284 35 -0.047244951129 -0.000258310997682 0.0069833729575\n",
      "284 40 -0.0444151759148 -0.000258310997682 0.00698263406535\n",
      "284 45 -0.0466134510934 -0.000258310997682 0.00698184621473\n",
      "284 50 -0.0419799499214 -0.000258310997682 0.00698108962355\n",
      "284 55 -0.0488935634494 -0.000258310997682 0.00698032686592\n",
      "valid_acc 98.35000000000001\n",
      "285 0 -0.0539209656417 -0.000258310997682 0.00697963087822\n",
      "285 5 -0.0433770231903 -0.000258310997682 0.00697893319697\n",
      "285 10 -0.0407298989594 -0.000258310997682 0.00697819072977\n",
      "285 15 -0.0572475045919 -0.000258310997682 0.00697742414251\n",
      "285 20 -0.0438584983349 -0.000258310997682 0.00697668295566\n",
      "285 25 -0.0424975454807 -0.000258310997682 0.00697578637995\n",
      "285 30 -0.0516046956182 -0.000258310997682 0.0069748984591\n",
      "285 35 -0.0362137369812 -0.000258310997682 0.00697417207756\n",
      "285 40 -0.0499780289829 -0.000258310997682 0.00697335094061\n",
      "285 45 -0.0409438423812 -0.000258310997682 0.00697241950356\n",
      "285 50 -0.0329700633883 -0.000258310997682 0.00697141228558\n",
      "285 55 -0.0380746275187 -0.000258310997682 0.00697035368509\n",
      "valid_acc 98.33\n",
      "286 0 -0.0591529160738 -0.000258310997682 0.00696932264804\n",
      "286 5 -0.0760811418295 -0.000258310997682 0.00696845106784\n",
      "286 10 -0.0276776198298 -0.000258310997682 0.00696754287979\n",
      "286 15 -0.0549345873296 -0.000258310997682 0.0069666522782\n",
      "286 20 -0.0625962838531 -0.000258310997682 0.00696591618186\n",
      "286 25 -0.0610744506121 -0.000258310997682 0.00696531426284\n",
      "286 30 -0.0405838638544 -0.000258310997682 0.00696481555611\n",
      "286 35 -0.0407690890133 -0.000258310997682 0.00696427958612\n",
      "286 40 -0.0523669868708 -0.000258310997682 0.00696352292912\n",
      "286 45 -0.0344392359257 -0.000258310997682 0.00696243094728\n",
      "286 50 -0.0461239516735 -0.000258310997682 0.00696134443756\n",
      "286 55 -0.0406316183507 -0.000258310997682 0.00696010996435\n",
      "valid_acc 98.315\n",
      "287 0 -0.0396056510508 -0.000258310997682 0.00695892313967\n",
      "287 5 -0.0487778000534 -0.000258310997682 0.00695781390511\n",
      "287 10 -0.0477822087705 -0.000258310997682 0.00695677355057\n",
      "287 15 -0.056181691587 -0.000258310997682 0.00695567500018\n",
      "287 20 -0.0408694222569 -0.000258310997682 0.00695440575785\n",
      "287 25 -0.0514096803963 -0.000258310997682 0.00695289298018\n",
      "287 30 -0.0937714725733 -0.000258310997682 0.00695158749994\n",
      "287 35 -0.0530378781259 -0.000258310997682 0.00695034398663\n",
      "287 40 -0.0631489083171 -0.000258310997682 0.006949048051\n",
      "287 45 -0.0723541006446 -0.000258310997682 0.0069477148949\n",
      "287 50 -0.0769666135311 -0.000258310997682 0.00694628061131\n",
      "287 55 -0.0391329340637 -0.000258310997682 0.00694476362496\n",
      "valid_acc 98.29666666666667\n",
      "288 0 -0.028118500486 -0.000258310997682 0.00694313257768\n",
      "288 5 -0.0579594038427 -0.000258310997682 0.00694137061085\n",
      "288 10 -0.0498701930046 -0.000258310997682 0.0069394387693\n",
      "288 15 -0.0358482562006 -0.000258310997682 0.00693765886874\n",
      "288 20 -0.0549240335822 -0.000258310997682 0.0069359098236\n",
      "288 25 -0.065428070724 -0.000258310997682 0.00693406957184\n",
      "288 30 -0.0578615963459 -0.000258310997682 0.00693228633295\n",
      "288 35 -0.0621150396764 -0.000258310997682 0.00693069226769\n",
      "288 40 -0.0700270906091 -0.000258310997682 0.00692914273429\n",
      "288 45 -0.0619025118649 -0.000258310997682 0.00692755660193\n",
      "288 50 -0.0564124509692 -0.000258310997682 0.00692612194765\n",
      "288 55 -0.0429286509752 -0.000258310997682 0.0069247791806\n",
      "valid_acc 98.335\n",
      "289 0 -0.0501733981073 -0.000258310997682 0.00692334919983\n",
      "289 5 -0.0538827553391 -0.000258310997682 0.00692196720045\n",
      "289 10 -0.0537105724216 -0.000258310997682 0.00692082515765\n",
      "289 15 -0.0427061654627 -0.000258310997682 0.00691986628173\n",
      "289 20 -0.0663054063916 -0.000258310997682 0.00691910978612\n",
      "289 25 -0.0513409040868 -0.000258310997682 0.0069184679089\n",
      "289 30 -0.058014113456 -0.000258310997682 0.0069177963278\n",
      "289 35 -0.0614289157093 -0.000258310997682 0.00691722389404\n",
      "289 40 -0.0511928647757 -0.000258310997682 0.00691693627807\n",
      "289 45 -0.0399485640228 -0.000258310997682 0.0069166994927\n",
      "289 50 -0.0327662564814 -0.000258310997682 0.00691645627193\n",
      "289 55 -0.0651726201177 -0.000258310997682 0.00691634074761\n",
      "valid_acc 98.34833333333334\n",
      "290 0 -0.0519259311259 -0.000258310997682 0.00691617243411\n",
      "290 5 -0.0381704978645 -0.000258310997682 0.00691582049883\n",
      "290 10 -0.0468055233359 -0.000258310997682 0.0069154036637\n",
      "290 15 -0.0469933487475 -0.000258310997682 0.00691493130039\n",
      "290 20 -0.0565897524357 -0.000258310997682 0.00691445086459\n",
      "290 25 -0.0514086447656 -0.000258310997682 0.00691407344562\n",
      "290 30 -0.048614051193 -0.000258310997682 0.00691364504481\n",
      "290 35 -0.0510796643794 -0.000258310997682 0.00691297661784\n",
      "290 40 -0.0467747636139 -0.000258310997682 0.00691235134531\n",
      "290 45 -0.059969227761 -0.000258310997682 0.00691178988838\n",
      "290 50 -0.0696744248271 -0.000258310997682 0.00691121692781\n",
      "290 55 -0.042141020298 -0.000258310997682 0.00691070399698\n",
      "valid_acc 98.36\n",
      "291 0 -0.0340034924448 -0.000258310997682 0.00691015337405\n",
      "291 5 -0.0981739535928 -0.000258310997682 0.00690973711641\n",
      "291 10 -0.0431501753628 -0.000258310997682 0.00690923674157\n",
      "291 15 -0.0492324717343 -0.000258310997682 0.00690873395675\n",
      "291 20 -0.0464767552912 -0.000258310997682 0.00690829991967\n",
      "291 25 -0.0290788188577 -0.000258310997682 0.00690777994867\n",
      "291 30 -0.0513074547052 -0.000258310997682 0.00690704495727\n",
      "291 35 -0.0566722005606 -0.000258310997682 0.00690632824252\n",
      "291 40 -0.0607614070177 -0.000258310997682 0.00690567600834\n",
      "291 45 -0.0702841728926 -0.000258310997682 0.00690505798633\n",
      "291 50 -0.043342217803 -0.000258310997682 0.00690439336376\n",
      "291 55 -0.0581752024591 -0.000258310997682 0.00690392010717\n",
      "valid_acc 98.32499999999999\n",
      "292 0 -0.0388437882066 -0.000258310997682 0.00690333920536\n",
      "292 5 -0.0450914204121 -0.000258310997682 0.00690274795587\n",
      "292 10 -0.0451919622719 -0.000258310997682 0.00690217294289\n",
      "292 15 -0.0749068856239 -0.000258310997682 0.00690163162545\n",
      "292 20 -0.0538670755923 -0.000258310997682 0.00690117872284\n",
      "292 25 -0.0384127870202 -0.000258310997682 0.00690066120589\n",
      "292 30 -0.0557535849512 -0.000258310997682 0.00689996578135\n",
      "292 35 -0.0233992617577 -0.000258310997682 0.00689940899322\n",
      "292 40 -0.0654509440064 -0.000258310997682 0.00689885203883\n",
      "292 45 -0.0577709972858 -0.000258310997682 0.00689827171428\n",
      "292 50 -0.0715618729591 -0.000258310997682 0.00689768205745\n",
      "292 55 -0.0711694657803 -0.000258310997682 0.00689715839155\n",
      "valid_acc 98.36333333333333\n",
      "best valid_acc 98.36333333333333\n",
      "293 0 -0.0446565411985 -0.000258310997682 0.00689657442045\n",
      "293 5 -0.0471897423267 -0.000258310997682 0.00689590792652\n",
      "293 10 -0.0526331365108 -0.000258310997682 0.00689537912732\n",
      "293 15 -0.0492459237576 -0.000258310997682 0.00689494876625\n",
      "293 20 -0.0318826548755 -0.000258310997682 0.00689446714777\n",
      "293 25 -0.05361860618 -0.000258310997682 0.00689382279702\n",
      "293 30 -0.039509113878 -0.000258310997682 0.00689316411061\n",
      "293 35 -0.057220466435 -0.000258310997682 0.006892596519\n",
      "293 40 -0.0353111512959 -0.000258310997682 0.006892111701\n",
      "293 45 -0.0504243560135 -0.000258310997682 0.00689150761488\n",
      "293 50 -0.044558968395 -0.000258310997682 0.00689077615184\n",
      "293 55 -0.0492988601327 -0.000258310997682 0.00689005825025\n",
      "valid_acc 98.34333333333333\n",
      "294 0 -0.0547527745366 -0.000258310997682 0.00688941879704\n",
      "294 5 -0.0492517873645 -0.000258310997682 0.00688883881444\n",
      "294 10 -0.0566680468619 -0.000258310997682 0.0068880910688\n",
      "294 15 -0.0693032369018 -0.000258310997682 0.00688735466012\n",
      "294 20 -0.0830789282918 -0.000258310997682 0.00688672727589\n",
      "294 25 -0.0365990623832 -0.000258310997682 0.00688617471223\n",
      "294 30 -0.0482201091945 -0.000258310997682 0.00688553035867\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "294 35 -0.0473731011152 -0.000258310997682 0.00688492123043\n",
      "294 40 -0.0527782142162 -0.000258310997682 0.00688439590311\n",
      "294 45 -0.0531230457127 -0.000258310997682 0.00688380220108\n",
      "294 50 -0.0585655719042 -0.000258310997682 0.00688304407878\n",
      "294 55 -0.065262503922 -0.000258310997682 0.00688229762213\n",
      "valid_acc 98.32\n",
      "295 0 -0.0415292531252 -0.000258310997682 0.00688151382898\n",
      "295 5 -0.0570113435388 -0.000258310997682 0.0068806082459\n",
      "295 10 -0.0493559762836 -0.000258310997682 0.00687960478319\n",
      "295 15 -0.0502278879285 -0.000258310997682 0.00687874584947\n",
      "295 20 -0.0378732942045 -0.000258310997682 0.00687809531653\n",
      "295 25 -0.0482927560806 -0.000258310997682 0.00687736866061\n",
      "295 30 -0.0344766080379 -0.000258310997682 0.00687675091138\n",
      "295 35 -0.0550314337015 -0.000258310997682 0.0068762223969\n",
      "295 40 -0.0354993529618 -0.000258310997682 0.0068756927421\n",
      "295 45 -0.0652955248952 -0.000258310997682 0.00687527940141\n",
      "295 50 -0.0592382065952 -0.000258310997682 0.0068750552164\n",
      "295 55 -0.0608885735273 -0.000258310997682 0.00687473390393\n",
      "valid_acc 98.29333333333334\n",
      "296 0 -0.0779951512814 -0.000258310997682 0.00687429548948\n",
      "296 5 -0.0560196787119 -0.000258310997682 0.00687403863735\n",
      "296 10 -0.042562186718 -0.000258310997682 0.00687377376429\n",
      "296 15 -0.0413898229599 -0.000258310997682 0.00687350079659\n",
      "296 20 -0.0426565632224 -0.000258310997682 0.00687325458439\n",
      "296 25 -0.0510987080634 -0.000258310997682 0.00687304735879\n",
      "296 30 -0.0477923117578 -0.000258310997682 0.00687273715761\n",
      "296 35 -0.0359129346907 -0.000258310997682 0.00687245588442\n",
      "296 40 -0.0493051595986 -0.000258310997682 0.00687228588526\n",
      "296 45 -0.0334103815258 -0.000258310997682 0.00687209583208\n",
      "296 50 -0.0802367925644 -0.000258310997682 0.00687189381956\n",
      "296 55 -0.0254645012319 -0.000258310997682 0.00687179507706\n",
      "valid_acc 98.315\n",
      "297 0 -0.0523705855012 -0.000258310997682 0.00687157090574\n",
      "297 5 -0.0639751031995 -0.000258310997682 0.006871376483\n",
      "297 10 -0.0437550470233 -0.000258310997682 0.00687106558577\n",
      "297 15 -0.0654457211494 -0.000258310997682 0.00687086188838\n",
      "297 20 -0.0846140831709 -0.000258310997682 0.00687067452086\n",
      "297 25 -0.0546525493264 -0.000258310997682 0.00687051507155\n",
      "297 30 -0.0484667420387 -0.000258310997682 0.00687048093418\n",
      "297 35 -0.0315280333161 -0.000258310997682 0.0068705954883\n",
      "297 40 -0.0557191520929 -0.000258310997682 0.00687064476446\n",
      "297 45 -0.0435185022652 -0.000258310997682 0.00687058431317\n",
      "297 50 -0.02761833556 -0.000258310997682 0.0068703340435\n",
      "297 55 -0.0441185608506 -0.000258310997682 0.00686998722368\n",
      "valid_acc 98.35166666666667\n",
      "298 0 -0.0702574849129 -0.000258310997682 0.00686954167054\n",
      "298 5 -0.0701278150082 -0.000258310997682 0.00686906238249\n",
      "298 10 -0.0642030984163 -0.000258310997682 0.00686850731643\n",
      "298 15 -0.0645646750927 -0.000258310997682 0.00686798446363\n",
      "298 20 -0.0375283360481 -0.000258310997682 0.00686740586291\n",
      "298 25 -0.0484636686742 -0.000258310997682 0.0068668204034\n",
      "298 30 -0.057812359184 -0.000258310997682 0.00686622481216\n",
      "298 35 -0.0419541411102 -0.000258310997682 0.00686559445813\n",
      "298 40 -0.0362670905888 -0.000258310997682 0.00686496929795\n",
      "298 45 -0.059227000922 -0.000258310997682 0.0068644580802\n",
      "298 50 -0.0689462944865 -0.000258310997682 0.00686403263743\n",
      "298 55 -0.0413454957306 -0.000258310997682 0.00686365911771\n",
      "valid_acc 98.35666666666667\n",
      "299 0 -0.0835369303823 -0.000258310997682 0.00686336207457\n",
      "299 5 -0.0436675474048 -0.000258310997682 0.00686322235318\n",
      "299 10 -0.0291300229728 -0.000258310997682 0.00686326406874\n",
      "299 15 -0.0598699264228 -0.000258310997682 0.00686354332299\n",
      "299 20 -0.039190184325 -0.000258310997682 0.00686382559577\n",
      "299 25 -0.0452866144478 -0.000258310997682 0.00686404327627\n",
      "299 30 -0.0330731421709 -0.000258310997682 0.00686413261406\n",
      "299 35 -0.047778069973 -0.000258310997682 0.006864130767\n",
      "299 40 -0.0397954620421 -0.000258310997682 0.00686408287548\n",
      "299 45 -0.0465805307031 -0.000258310997682 0.00686402754236\n",
      "299 50 -0.0583797357976 -0.000258310997682 0.0068640253313\n",
      "299 55 -0.0616753697395 -0.000258310997682 0.00686398442388\n",
      "valid_acc 98.26666666666667\n",
      "300 0 -0.0459624715149 -0.000258310997682 0.00686384355946\n",
      "300 5 -0.0543454326689 -0.000258310997682 0.00686343317882\n",
      "300 10 -0.0329040661454 -0.000258310997682 0.00686297006886\n",
      "300 15 -0.0536524169147 -0.000258310997682 0.00686253966845\n",
      "300 20 -0.057625759393 -0.000258310997682 0.00686200746156\n",
      "300 25 -0.0388411656022 -0.000258310997682 0.00686125389484\n",
      "300 30 -0.0919260755181 -0.000258310997682 0.0068605193768\n",
      "300 35 -0.0628586784005 -0.000258310997682 0.00685979538454\n",
      "300 40 -0.0281958319247 -0.000258310997682 0.00685902011916\n",
      "300 45 -0.040229331702 -0.000258310997682 0.00685819047051\n",
      "300 50 -0.0408541113138 -0.000258310997682 0.00685737760085\n",
      "300 55 -0.0640824735165 -0.000258310997682 0.00685635633479\n",
      "valid_acc 98.3\n"
     ]
    }
   ],
   "source": [
    "#'''\n",
    "best_valid_acc = 0\n",
    "training_log = []\n",
    "for epoch in range(1, 10*args.epochs + 1):\n",
    "\n",
    "  # train loop\n",
    "  model.eval()\n",
    "  for batch_idx, (data, target) in enumerate(train_loader):\n",
    "    if args.cuda:\n",
    "      data, target = data.cuda(), target.cuda()\n",
    "    data, target = Variable(data), Variable(target)\n",
    "    \n",
    "    solutions = es.ask()\n",
    "    reward = np.zeros(es.popsize)\n",
    "    \n",
    "    for i in range(es.popsize):\n",
    "      update_model(solutions[i], model, model_shapes)\n",
    "      output = model(data)\n",
    "      loss = F.nll_loss(output, target)\n",
    "      reward[i] = - loss.data[0]\n",
    "\n",
    "    best_raw_reward = reward.max()\n",
    "    #reward = compute_centered_ranks(reward)\n",
    "    #l2_decay = compute_weight_decay(weight_decay_coef, solutions)\n",
    "    #reward += l2_decay\n",
    "\n",
    "    es.tell(reward)\n",
    "\n",
    "    result = es.result()\n",
    "    \n",
    "    if (batch_idx % 5 == 0):\n",
    "      print(epoch, batch_idx, best_raw_reward, result[0].mean(), result[3].mean())\n",
    "\n",
    "  curr_solution = es.current_param()\n",
    "  update_model(curr_solution, model, model_shapes)\n",
    "\n",
    "  valid_acc = evaluate(model, valid_loader, print_mode=False)\n",
    "  training_log.append([epoch, valid_acc])\n",
    "  print('valid_acc', valid_acc * 100.)\n",
    "  if valid_acc >= best_valid_acc:\n",
    "    best_valid_acc = valid_acc\n",
    "    best_model = copy.deepcopy(model)\n",
    "    print('best valid_acc', best_valid_acc * 100.)\n",
    "#'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Average loss: 0.0540, Accuracy: 59018/60000 (98.3633%)\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9836333333333334"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate(best_model, valid_loader, print_mode=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Average loss: 0.0678, Accuracy: 9813/10000 (98.1300%)\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9813"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate(best_model, test_loader, print_mode=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Average loss: 0.0540, Accuracy: 59018/60000 (98.3633%)\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9836333333333334"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate(best_model, train_loader, print_mode=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "update_model(es.best_param(), model, model_shapes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Average loss: 0.0627, Accuracy: 58864/60000 (98.1067%)\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9810666666666666"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate(model, valid_loader, print_mode=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Average loss: 0.0772, Accuracy: 9790/10000 (97.9000%)\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.979"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate(model, test_loader, print_mode=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Average loss: 0.0627, Accuracy: 58864/60000 (98.1067%)\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9810666666666666"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate(model, train_loader, print_mode=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "update_model(es.current_param(), model, model_shapes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Average loss: 0.0556, Accuracy: 58980/60000 (98.3000%)\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.983"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate(model, valid_loader, print_mode=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Average loss: 0.0684, Accuracy: 9799/10000 (97.9900%)\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9799"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate(model, test_loader, print_mode=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Average loss: 0.0556, Accuracy: 58980/60000 (98.3000%)\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.983"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate(model, train_loader, print_mode=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Average loss: 0.0678, Accuracy: 9813/10000 (98.1300%)\n",
      "\n",
      "final test acc 98.13\n"
     ]
    }
   ],
   "source": [
    "eval_acc = evaluate(best_model, test_loader)\n",
    "print('final test acc', eval_acc * 100.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([8, 1, 5, 5])\n",
      "torch.Size([8])\n",
      "torch.Size([16, 8, 5, 5])\n",
      "torch.Size([16])\n",
      "torch.Size([10, 784])\n",
      "torch.Size([10])\n",
      "11274\n"
     ]
    }
   ],
   "source": [
    "param_count = 0\n",
    "for param in model.parameters():\n",
    "  print(param.data.shape)\n",
    "  param_count += np.product(param.data.shape)\n",
    "print(param_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "orig_params = []\n",
    "for param in orig_model.parameters():\n",
    "  orig_params.append(param.data.cpu().numpy().flatten())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "orig_params_flat = np.concatenate(orig_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAEexJREFUeJzt3X+IZWd9x/H3xySmRS1NmnFdN2snwrZlU2qUaZQqRWtr\nYvxjI5SwodiFpqxCFAX9Y6N/aJGFtFSFQpWuGtwWNV0wksWkLckiiKiJE4lJNjFmNRuyyya7/qr2\nn7RZv/1jTvQ6zsw9d+69OzPPvF9wmXOf8zznfOfM3c+cee65Z1NVSJLa9by1LkCSNF0GvSQ1zqCX\npMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalx5691AQCXXHJJzc7OrnUZkrSh3HfffT+oqplh\n/dZF0M/OzjI/P7/WZUjShpLkiT79nLqRpMYZ9JLUOINekho3NOiT/EaSe5N8O8nRJH/XtV+c5K4k\nj3VfLxoYc1OSY0keTXLVNL8BSdLK+pzRPwP8WVW9ArgCuDrJa4B9wJGq2gEc6Z6TZCewG7gcuBr4\neJLzplG8JGm4oUFfC/6ne3pB9yhgF3Cwaz8IXNst7wJurapnqupx4Bhw5USrliT11muOPsl5Se4H\nTgN3VdU9wJaqOtV1eQrY0i1vA54cGH6ia5MkrYFeQV9VZ6vqCuBS4Mokf7hofbFwlt9bkr1J5pPM\nnzlzZpShkqQRjHTVTVX9BPgyC3PvTyfZCtB9Pd11OwlsHxh2ade2eFsHqmququZmZoZ+sEuStEp9\nrrqZSfLb3fJvAn8BfAc4DOzpuu0Bbu+WDwO7k1yY5DJgB3DvpAuXWjG77w5m992x1mWoYX1ugbAV\nONhdOfM84FBVfSnJ14FDSW4AngCuA6iqo0kOAQ8DzwI3VtXZ6ZQvSRpmaNBX1QPAK5do/yHwxmXG\n7Af2j12dJGlsfjJWOkecotFaMeglqXEGvTYNz6i1WRn0ktQ4g16bzrk8q/cvCK0HBr02JadxtJkY\n9GqG4S0tzaCXpMYZ9JLUOINekhpn0GtTmNTcve8BaCMy6LXhDHvT1TCWfpVBL3X6/oLw6h5tNAa9\nJDXOoJekxvX5j0ckjcipHa0nntFLa8hfCDoXPKOXejCQtZF5Ri8twWBXSwx6SWqcQS9JjXOOXpoy\np4G01jyjVxMMU2l5Br20jNXe6mDYGH8p6Vwz6CWpcQa9NjXPrrUZDA36JNuTfDnJw0mOJnl31/6h\nJCeT3N89rhkYc1OSY0keTXLVNL8BaSneYVL6pT5X3TwLvLeqvpXkRcB9Se7q1n2sqv5xsHOSncBu\n4HLgpcDdSX6vqs5OsnBpPZnddwfHb37LSP2lc2XoGX1Vnaqqb3XLPwMeAbatMGQXcGtVPVNVjwPH\ngCsnUawkaXQjzdEnmQVeCdzTNb0ryQNJbklyUde2DXhyYNgJlvjFkGRvkvkk82fOnBm5cGmteVau\njaJ30Cd5IfAF4D1V9VPgE8DLgSuAU8BHRtlxVR2oqrmqmpuZmRllqCRpBL2CPskFLIT8Z6vqNoCq\nerqqzlbVz4FP8svpmZPA9oHhl3ZtkqQ1MPTN2CQBPg08UlUfHWjfWlWnuqdvBR7qlg8Dn0vyURbe\njN0B3DvRqqUpcTpGLepz1c1rgbcBDya5v2t7P3B9kiuAAo4DbweoqqNJDgEPs3DFzo1ecSNJa2do\n0FfVV4EsserOFcbsB/aPUZe0IUzyL4BRL9GU+vLulWraJILY6RxtdN4CQZIaZ9Brw1pvZ9redkHr\nlVM32tAMVmk4z+glqXEGvSQ1zqkbbXpO/6h1ntFLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0\nktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9J\njRsa9Em2J/lykoeTHE3y7q794iR3JXms+3rRwJibkhxL8miSq6b5DUiSVtbnjP5Z4L1VtRN4DXBj\nkp3APuBIVe0AjnTP6dbtBi4HrgY+nuS8aRQvSRpuaNBX1amq+la3/DPgEWAbsAs42HU7CFzbLe8C\nbq2qZ6rqceAYcOWkC5ck9TPSHH2SWeCVwD3Alqo61a16CtjSLW8DnhwYdqJrW7ytvUnmk8yfOXNm\nxLIlSX31DvokLwS+ALynqn46uK6qCqhRdlxVB6pqrqrmZmZmRhkqSRpBr6BPcgELIf/Zqrqta346\nydZu/VbgdNd+Etg+MPzSrk2StAb6XHUT4NPAI1X10YFVh4E93fIe4PaB9t1JLkxyGbADuHdyJUuS\nRnF+jz6vBd4GPJjk/q7t/cDNwKEkNwBPANcBVNXRJIeAh1m4YufGqjo78colSb0MDfqq+iqQZVa/\ncZkx+4H9Y9QlSZoQPxkrSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+gl\nqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIa\nZ9BLUuMMeklqnEEvSY0z6CWpcUODPsktSU4neWig7UNJTia5v3tcM7DupiTHkjya5KppFS5J6qfP\nGf1ngKuXaP9YVV3RPe4ESLIT2A1c3o35eJLzJlWsJGl0Q4O+qr4C/Kjn9nYBt1bVM1X1OHAMuHKM\n+iRJYxpnjv5dSR7opnYu6tq2AU8O9DnRtf2aJHuTzCeZP3PmzBhlSJJWstqg/wTwcuAK4BTwkVE3\nUFUHqmququZmZmZWWYYkaZhVBX1VPV1VZ6vq58An+eX0zElg+0DXS7s2SdIaWVXQJ9k68PStwHNX\n5BwGdie5MMllwA7g3vFKlCSN4/xhHZJ8Hng9cEmSE8AHgdcnuQIo4DjwdoCqOprkEPAw8CxwY1Wd\nnU7pkqQ+hgZ9VV2/RPOnV+i/H9g/TlGSpMnxk7GS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9\nJDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS\n4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1LihQZ/kliSnkzw00HZxkruSPNZ9vWhg3U1JjiV5\nNMlV0ypcktRPnzP6zwBXL2rbBxypqh3Ake45SXYCu4HLuzEfT3LexKqVJI1saNBX1VeAHy1q3gUc\n7JYPAtcOtN9aVc9U1ePAMeDKCdUqSVqF1c7Rb6mqU93yU8CWbnkb8ORAvxNdmyRpjYz9ZmxVFVCj\njkuyN8l8kvkzZ86MW4YkaRmrDfqnk2wF6L6e7tpPAtsH+l3atf2aqjpQVXNVNTczM7PKMiRJw6w2\n6A8De7rlPcDtA+27k1yY5DJgB3DveCVKksZx/rAOST4PvB64JMkJ4IPAzcChJDcATwDXAVTV0SSH\ngIeBZ4Ebq+rslGqXJPUwNOir6vplVr1xmf77gf3jFCVJmhw/GStJjTPoJalxBr0kNc6gl6TGGfSS\n1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mN\nM+glqXEGvSQ1zqDXOTe77461LkHaVAx6SWqcQS81wL+StBKDXhM3u+8Og0daRwx6SWqcQa8NYbP9\nlTCN7/W5Y7iZjqMWGPTaUAwpaXTnjzM4yXHgZ8BZ4NmqmktyMfDvwCxwHLiuqn48XpmSRvHcL8Tj\nN79ljSvRejCJM/o3VNUVVTXXPd8HHKmqHcCR7rkkaY1MY+pmF3CwWz4IXDuFfUiSehpr6gYo4O4k\nZ4F/qaoDwJaqOtWtfwrYMuY+JPWw3HSN72to3KB/XVWdTPJi4K4k3xlcWVWVpJYamGQvsBfgZS97\n2ZhlSFqKIS8Yc+qmqk52X08DXwSuBJ5OshWg+3p6mbEHqmququZmZmbGKUPrlCEzGR5HjWvVQZ/k\nBUle9Nwy8CbgIeAwsKfrtge4fdwiJUmrN87UzRbgi0me287nquo/k3wTOJTkBuAJ4Lrxy9RG1Wfe\neLlLADfjJYKDx8UzeU3KqoO+qr4PvGKJ9h8CbxynKG0umzHQpXPJT8Zq3fAMVpoOg17aYIbdr8Zf\nmFps3MsrpV8YFjAG0GTN7rvD6S71YtBLG5i/PNWHUzfSBmKwazUMep0TowaUgSZNjlM3WlcM+KV5\nXDSOVC15K5pzam5urubn59e6DI3BINqYfDN3Y0ty38At4pfl1I0kNc6gl6TGGfSS1DiDXpIaZ9BL\nUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1\nzqCXpMZNLeiTXJ3k0STHkuyb1n4kSSubStAnOQ/4Z+DNwE7g+iQ7p7EvSdoI1vL/VZ7WGf2VwLGq\n+n5V/S9wK7BrSvv6tQM4u++OJdv6jO2zr6W2v9T2BvuuZv8rjetTSx/j7L9vH61fi1+vK/Vbav1q\nf/4r/Xvou70+/74Wb3OUvkvtZ9z+a+X8KW13G/DkwPMTwKuntC9gvN+WS409fvNbfqX9+M1vGXuf\ny21v2H4G+4yz/rl1g8uL97/autWGwdfRUq+pxa+dpdpHfa31fW0vrqtP7X0s9z2N0nel9pVqheW/\n70lKVU1+o8lfAldX1d92z98GvLqq3jnQZy+wt3v6+8Cjq9jVJcAPxix3WtZrbdY1uvVa23qtC9Zv\nba3V9btVNTOs07TO6E8C2weeX9q1/UJVHQAOjLOTJPNVNTfONqZlvdZmXaNbr7Wt17pg/da2Weua\n1hz9N4EdSS5L8nxgN3B4SvuSJK1gKmf0VfVskncC/wWcB9xSVUensS9J0sqmNXVDVd0J3Dmt7XfG\nmvqZsvVam3WNbr3Wtl7rgvVb26asaypvxkqS1g9vgSBJjVv3QZ/k4iR3JXms+3rREn22J/lykoeT\nHE3y7lHGT6uurt8tSU4neWhR+4eSnExyf/e4ZhJ1Tai2tT5mS94+Y9LHbNhtOrLgn7r1DyR5Vd+x\n4xqztuNJHuyO0fw5rusPknw9yTNJ3jfK2DWsa2rHq2dtf9X9DB9M8rUkr+g7treqWtcP4B+Afd3y\nPuDvl+izFXhVt/wi4LvAzr7jp1VXt+5PgVcBDy1q/xDwvrU6ZkNqW7NjxsKb998DXg48H/j2wM9y\nYsdspf0M9LkG+A8gwGuAe/qOXavaunXHgUum8LrqU9eLgT8G9g/+rKZ5zMapa5rHa4Ta/gS4qFt+\n8zReZ+v+jJ6FWycc7JYPAtcu7lBVp6rqW93yz4BHWPh0bq/x06qrq+crwI8mtM++xq1tLY/Zubp9\nRp/97AL+tRZ8A/jtJFvPQY3j1DZNQ+uqqtNV9U3g/0Ydu0Z1TVuf2r5WVT/unn6Dhc8d9Rrb10YI\n+i1VdapbfgrYslLnJLPAK4F7VjN+WnUt413dn2y3TGp6ZEK1reUxW+r2GdsGnk/qmA3bz0p9+owd\nxzi1ARRwd5L7svAJ9HNZ1zTGTnvb0zpeMHptN7Dwl9pqxi5rapdXjiLJ3cBLllj1gcEnVVVJlr1M\nKMkLgS8A76mqny5eP2z8tOpaxieAD7PwIvsw8BHgb9ZJbasev56P2Sbyuqo6meTFwF1JvtP99aal\nrYvjleQNLAT96ya97XUR9FX158utS/J0kq1Vdar70/T0Mv0uYCHkP1tVtw2s6jV+WnWtsO2nB7b1\nSeBLI46fWm2s7TFb9vYZ4x6zvvvp0eeCHmPHMU5tVNVzX08n+SILUwCTCK4+dU1j7FS3PcXj1bu2\nJH8EfAp4c1X9cJSxfWyEqZvDwJ5ueQ9w++IOSQJ8Gnikqj466vhp1bWSRfOpbwUeWq7vKoz7Pa/l\nMVv29hkTPmZ9btNxGPjr7gqX1wD/3U09TfsWH6uuLckLkrwIIMkLgDcxudfWON/3NI/Zqrc95ePV\nq7YkLwNuA95WVd8dZWxv03ineZIP4HeAI8BjwN3AxV37S4E7u+XXsfDn/APA/d3jmpXGn4u6uuef\nB06x8CbQCeCGrv3fgAe7mg8DW8/lMRtS21ofs2tYuHLqe8AHBtonesyW2g/wDuAd3XJY+A90vtft\nd25YjRP8Ga6qNhau0Ph29zg66dp61PWS7rX0U+An3fJvTfuYrbauaR+vnrV9Cvgxv8yu+Um/zvxk\nrCQ1biNM3UiSxmDQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUuP8Hm+2DKUyf1DkAAAAA\nSUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f778eafceb8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "_ = plt.hist(orig_params_flat, bins=200)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "final_params = []\n",
    "for param in best_model.parameters():\n",
    "  final_params.append(param.data.cpu().numpy().flatten())\n",
    "final_params_flat = np.concatenate(final_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFMJJREFUeJzt3X2QJPV93/H3JyBhPdmAWV/OwGUhhZSglHWSN1iJHoKN\nEiFwGSnlUiAWQgrOiYpMSbEq9iFVIidVVBFHD3bKkVwnQcCxBMICSUTgRAjLIi4L5EXB6HgSBxzi\nLge3BtlSKS6cg2/+mF48rHZvZ3emd2Z636+qre35dffM9/Z6Pv2b3/RDqgpJUnf9jXEXIElql0Ev\nSR1n0EtSxxn0ktRxBr0kdZxBL0kdZ9BLUscZ9JLUcQa9JHXckeMuAOC4446r2dnZcZchSVPljjvu\n+LOqmlltuYkI+tnZWebn58ddhiRNlSSPDLKcQzeS1HEGvSR1nEEvSR1n0EtSx60a9ElOTPKVJPck\nuTvJe5r2Y5PcnOSB5vcxfetckmRPkvuTvLHNf4Ak6fAG6dEfAt5XVacCrwbeneRUYCdwS1WdAtzS\nPKaZdy7wcuBM4GNJjmijeEnS6lYN+qo6UFXfaKa/B9wLHA+cA1zVLHYV8OZm+hzgmqp6qqoeBvYA\np426cEnSYNY0Rp9kFnglcDuwpaoONLMeA7Y008cDj/attq9pkySNwcBBn+TFwHXAe6vqu/3zqnfj\n2TXdfDbJjiTzSeYXFhbWsqokaQ0GCvokz6MX8p+qquub5seTbG3mbwUONu37gRP7Vj+haXuOqtpV\nVXNVNTczs+oZvFJrZnfeyOzOG8ddhtSaQY66CXA5cG9VfaRv1g3ABc30BcAX+trPTXJUkpOAU4Cv\nj65kSdJaDHKtm9cA5wPfTHJn0/Z+4DLg2iQXAo8AbwWoqruTXAvcQ++InXdX1dMjr1ySNJBVg76q\n/gjICrPPWGGdS4FLh6hLkjQinhkrSR1n0EtSxxn0ktRxBr20Th6SqWlh0EtSxxn0ktRxBr20As+Y\nVVcY9JLUcQa9NAB795pmg1wCQdrUlga8ga9pY49ekjrOoJekjjPoJanjDHpJ6jiDXpI6zqCXpI4z\n6CWp4wa5Z+wVSQ4m2d3X9pkkdzY/exdvMZhkNslf9s377TaLlyStbpATpq4Efgv4ncWGqvpni9NJ\nPgz8Rd/yD1bV9lEVKG0kT4ZSFw1yz9hbk8wuNy9J6N0U/GdGW5YkaVSGHaN/HfB4VT3Q13ZSM2zz\n1SSvG/L5pQ1jb15dNWzQnwdc3ff4ALCtGbr5ZeDTSX54uRWT7Egyn2R+YWFhyDKkwY0y0L3YmabB\nuoM+yZHAPwU+s9hWVU9V1RPN9B3Ag8BLl1u/qnZV1VxVzc3MzKy3DGldDGhtJsNcvfINwH1VtW+x\nIckM8GRVPZ3kZOAU4KEha5RGYhTB7s5B02iQwyuvBr4GvCzJviQXNrPO5bnDNgCvB+5qDrf8LHBR\nVT05yoIlSWszyFE3563Q/o5l2q4Drhu+LEnSqHhmrCR1nEEvSR1n0KuTlh5V45eo2swMeknqOINe\nm5o9fW0GBr0kdZxBL42YnxI0aQx6Seo4g15Tx+vUSGtj0KsTxh3843596XAMeknqOINekjpumMsU\nS+rj8I0mlT16Seo4e/SaavaipdXZo5ekjjPoJanjBrmV4BVJDibZ3df2a0n2J7mz+Tmrb94lSfYk\nuT/JG9sqXJpkntSlSTJIj/5K4Mxl2j9aVdubn5sAkpxK716yL2/W+ViSI0ZVrCRp7VYN+qq6FRj0\nBt/nANdU1VNV9TCwBzhtiPqkqWfvXuM2zBj9xUnuaoZ2jmnajgce7VtmX9MmSRqT9Qb9x4GTge3A\nAeDDa32CJDuSzCeZX1hYWGcZkqTVrCvoq+rxqnq6qp4BPsFfD8/sB07sW/SEpm2559hVVXNVNTcz\nM7OeMqTncIhEWt66gj7J1r6HbwEWj8i5ATg3yVFJTgJOAb4+XImSpGGsemZskquB04HjkuwDPgic\nnmQ7UMBe4F0AVXV3kmuBe4BDwLur6ul2StdmsthT33vZ2WOuRJo+qwZ9VZ23TPPlh1n+UuDSYYqS\nRmXSh3Jmd97ozkut88xYTa1JD3GYjhrVfQa9JHWcQS9JHWfQa6o4FCKtndejl8bAHZY2kj16Seo4\ng16aIJ7dqzYY9Oocg1J6LoNekjrOoJc2mJ84tNEMek08g1EajodXShvEHZbGxR69NCHcEagtBr0k\ndZxBL0kdZ9BLUscZ9JLUcasGfZIrkhxMsruv7T8luS/JXUk+l+Topn02yV8mubP5+e02i5ckrW6Q\nHv2VwJlL2m4G/l5V/QTwLeCSvnkPVtX25uei0ZSpzcJrvUijt2rQV9WtwJNL2r5UVYeah7cBJ7RQ\nm7RpubPTKI1ijP5fAL/f9/ikZtjmq0leN4Ln1yZk0EmjM9SZsUk+ABwCPtU0HQC2VdUTSX4S+HyS\nl1fVd5dZdwewA2Dbtm3DlCFJOox19+iTvAP4WeAXqqoAquqpqnqimb4DeBB46XLrV9WuqpqrqrmZ\nmZn1liFJWsW6evRJzgR+BfhHVfV/+9pngCer6ukkJwOnAA+NpFKpoxymUttWDfokVwOnA8cl2Qd8\nkN5RNkcBNycBuK05wub1wH9I8v+AZ4CLqurJZZ9YaiwG3d7Lzh5zJdPBv5fWatWgr6rzlmm+fIVl\nrwOuG7YoSX9tdueNhrqG4pmxktRxBr0kdZw3HpEmlF/SalTs0UtSxxn0ktRxBr0kdZxBL0kdZ9BL\nUscZ9JLUcQa9xqr/EEIPJ1ydfyOth0EvSR3nCVOaWPZepdEw6LUh+kPbC3RJG8uhG0nqOINeG84h\nGWljGfSS1HEGvSR13KpBn+SKJAeT7O5rOzbJzUkeaH4f0zfvkiR7ktyf5I1tFS5JGswgPforgTOX\ntO0EbqmqU4BbmsckORU4F3h5s87HkhwxsmolPcvvOjSoQe4Ze2uS2SXN59C7YTjAVcAfAr/atF9T\nVU8BDyfZA5wGfG005aorDClp46x3jH5LVR1oph8DtjTTxwOP9i23r2mTJI3J0F/GVlUBtdb1kuxI\nMp9kfmFhYdgyJEkrWO+ZsY8n2VpVB5JsBQ427fuBE/uWO6Fp+wFVtQvYBTA3N7fmHYWmg0M00vit\nt0d/A3BBM30B8IW+9nOTHJXkJOAU4OvDlShJGsaqPfokV9P74vW4JPuADwKXAdcmuRB4BHgrQFXd\nneRa4B7gEPDuqnq6pdolSQMY5Kib81aYdcYKy18KXDpMUZKeyyEwDcMzYyWp4wx6Seo4g16SOs6g\nl6SOM+ilKTa780a/qNWqDHpJ6jiDXpI6zqCXpI4z6NUax46lyWDQSx3gTlWHY9BLUset9zLF0ors\nXUqTxR69JHWcQS9JHWfQS1LHGfRSR3g5BK3EoNe6GSrSdFj3UTdJXgZ8pq/pZODfAUcD/xJYaNrf\nX1U3rbtCSWu2uBPee9nZY65Ek2DdQV9V9wPbAZIcAewHPge8E/hoVX1oJBVKWhM/aWmpUQ3dnAE8\nWFWPjOj5JEkjMqoTps4Fru57fHGStwPzwPuq6jsjeh1NmP7eo8ME0mQaukef5PnAzwG/1zR9nN54\n/XbgAPDhFdbbkWQ+yfzCwsJyi2jKeNSHNJlG0aN/E/CNqnocYPE3QJJPAF9cbqWq2gXsApibm6sR\n1KENYphL02UUY/Tn0Tdsk2Rr37y3ALtH8BqSpHUaqkef5EXAPwbe1df860m2AwXsXTJPkrTBhgr6\nqvo+8KNL2s4fqiJJ0kh5ZqwkdZxBL3WYX5wLDHoNyEMnp5f/dzLoJanjDHpJ6jiDXmviEIA0fQx6\nSeo4g14r8ks8qRsMemkTcee9ORn0ktRxBr0kdZxBL0kdZ9BLUscZ9JLUcQa9JHWcQS9JHWfQS1LH\nDXsrwb3A94CngUNVNZfkWOAzwCy9Wwm+taq+M1yZktrSfwLV3svO/oH2/jZNp1H06H+6qrZX1Vzz\neCdwS1WdAtzSPJY0QTxDdnNpY+jmHOCqZvoq4M0tvIakNTLYN69hg76ALye5I8mOpm1LVR1oph8D\ntgz5GpKkIQw1Rg+8tqr2J/kx4OYk9/XPrKpKUsut2OwYdgBs27ZtyDIkSSsZqkdfVfub3weBzwGn\nAY8n2QrQ/D64wrq7qmququZmZmaGKUOSdBjrDvokL0ryksVp4J8Au4EbgAuaxS4AvjBskZLa4bj9\n5jDM0M0W4HNJFp/n01X1P5L8CXBtkguBR4C3Dl+mNoqH1Ends+6gr6qHgFcs0/4EcMYwRUmSRscz\nYyWp4wx6rcpxXGm6GfSS1HEGvaRneWmEbjLotSzf7FqJO4PpY9BLUscZ9JtUf6/MHprUbQa9JHWc\nQS9JHWfQb3IO2Wg1biPTz6DXs3xDa5HbQrcY9JLUcQa9JHWcQS9pVR6CO90MeknqOINe0rrYw58e\nBr0kddww94w9MclXktyT5O4k72nafy3J/iR3Nj9nja5cSdJaDXPP2EPA+6rqG81Nwu9IcnMz76NV\n9aHhy1Mb/Mit9XLbmU7D3DP2AHCgmf5eknuB40dVmCRpNEYyRp9kFnglcHvTdHGSu5JckeSYUbyG\nJGl9hg76JC8GrgPeW1XfBT4OnAxsp9fj//AK6+1IMp9kfmFhYdgytAo/cqtNbl+TbaigT/I8eiH/\nqaq6HqCqHq+qp6vqGeATwGnLrVtVu6pqrqrmZmZmhilDS3hyizbKWrY1t8nxGeaomwCXA/dW1Uf6\n2rf2LfYWYPf6y9Mwlr6x3AFIm9MwR928Bjgf+GaSO5u29wPnJdkOFLAXeNdQFUqaCoudiL2XnT3m\nSrTUMEfd/BGQZWbdtP5yNGr24DVOsztvNPgngGfGSlLHGfQdYw9e0lIG/RTyS1VNMrfNyWPQS1LH\nDXPUjSQty179ZLFHL6lVyw01uiPYWPboJ8xKxyKvdoyybxxJK7FHP2UMdHWFBxVsHIN+zNzQJbXN\noJekjjPop8BKvX4/DUgahF/GTjnDXtPE7XU8DPoJ0v8m8A2hzcgrYLbDoRtJ6jh79BPA3rs2M3vx\n7bNHL2kirOWWhHaO1sagHwM3UunwRvkeccfQ4tBNkjOB3wSOAD5ZVZe19VqTpH+DOtxH0c2+4UmD\nOtzhxYvvsaV3snI46Lla6dEnOQL4L8CbgFPp3Uf21DZea5J5c25punT1PdpWj/40YE9VPQSQ5Brg\nHOCell6vFSv1zpf2Fg53X0x7FtL6DBK4yx2S3OZ7bVrvgdtW0B8PPNr3eB/wUy291g98hFvO4a4G\nOchwy3LPu5Y9fxd7CdIkOtx7bbn3fds7hkVrvSLtKKWqRv+kyc8DZ1bVLzaPzwd+qqp+qW+ZHcCO\n5uHLgPvX+DLHAX82gnJHaRJrAutaK+taG+tam1HW9beqama1hdrq0e8HTux7fELT9qyq2gXsWu8L\nJJmvqrn1rt+GSawJrGutrGttrGttxlFXW4dX/glwSpKTkjwfOBe4oaXXkiQdRis9+qo6lOSXgP9J\n7/DKK6rq7jZeS5J0eK0dR19VNwE3tfX8DDHs06JJrAmsa62sa22sa202vK5WvoyVJE0OL4EgSR03\nFUGf5NgkNyd5oPl9zArL/eskdyfZneTqJD80IXUdneSzSe5Lcm+SfzAJdTXLHpHkfyf5Yps1DVpX\nkhOTfCXJPc3/5XtarOfMJPcn2ZNk5zLzk+Q/N/PvSvKqtmpZY12/0NTzzSR/nOQVk1BX33J/P8mh\n5jDriagryelJ7my2qa+Ou6YkP5Lkvyf506amd7ZaUFVN/A/w68DOZnon8B+XWeZ44GHgBc3ja4F3\njLuuZt5VwC82088Hjp6Eupr5vwx8GvjihPw/bgVe1Uy/BPgWcGoLtRwBPAic3Pyf/OnS1wHOAn4f\nCPBq4PYN+BsNUtc/BI5ppt80KXX1LfcH9L6f+/lJqAs4mt5Z+duaxz82ATW9f3H7B2aAJ4Hnt1XT\nVPTo6V0+4apm+irgzSssdyTwgiRHAi8E/s+460ryI8DrgcsBquqvqurPx11XU9sJwNnAJ1uuZ+C6\nqupAVX2jmf4ecC+9nfioPXuZjqr6K2DxMh1L6/2d6rkNODrJ1hZqWVNdVfXHVfWd5uFt9M5Tadsg\nfy+Ai4HrgIMbUNOgdf1z4Pqq+jZAVbVd2yA1FfCSJAFeTC/oD7VV0LQE/ZaqOtBMPwZsWbpAVe0H\nPgR8GzgA/EVVfWncdQEnAQvAf22GSD6Z5EUTUBfAbwC/AjzTcj2LBq0LgCSzwCuB21uoZbnLdCzd\noQyyzDjq6nchvU8dbVu1riTHA28BPr4B9QxcF/BS4Jgkf5jkjiRvn4Cafgv4u/Q6o98E3lNVrb0P\nJ+YOU0m+DPzNZWZ9oP9BVVWSHzhUqBnvPYdesP458HtJ3lZVvzvOuuj9jV8FXFxVtyf5TXrDFv92\nnHUl+VngYFXdkeT0YWoZZV19z/Niej3D91bVd0dVX5ck+Wl6Qf/acdfS+A3gV6vqmV5HdWIcCfwk\ncAbwAuBrSW6rqm+NsaY3AncCPwP8beDmJP+rrW19YoK+qt6w0rwkjyfZWlUHmo/Oy330egPwcFUt\nNOtcT28sc6igH0Fd+4B9VbXYK/0svaAfygjqeg3wc0nOAn4I+OEkv1tVbxtzXSR5Hr2Q/1RVXT9M\nPYex6mU6BlxmHHWR5CfoDbm9qaqeaLmmQeuaA65pQv444Kwkh6rq82Ouax/wRFV9H/h+kluBV9D7\n/mdcNb0TuKx6g/R7kjwM/B3g620UNC1DNzcAFzTTFwBfWGaZbwOvTvLCZtzrDHrju2Otq6oeAx5N\n8rKm6Qzav1zzIHVdUlUnVNUsvUtU/MGwIT+Kupr/u8uBe6vqIy3WMshlOm4A3t4cffNqesOBB5Y+\n0UbXlWQbcD1w/gb2Sletq6pOqqrZZpv6LPCvWg75geqit529NsmRSV5I70q6bWbDIDV9m14WkGQL\nvQs7PtRaRW1++zyqH+BHgVuAB4AvA8c27T8O3NS33L8H7gN2A/8NOGpC6toOzAN3AZ+nOWJi3HX1\nLX86G3PUzap10RuGqOZvdWfzc1ZL9ZxFr1f3IPCBpu0i4KJmOvRuoPMgvXHUubb/RgPW9UngO31/\nn/lJqGvJsleyAUfdDFoX8G/odbB20xsOHPf/4Y8DX2q2q93A29qsxzNjJanjpmXoRpK0Tga9JHWc\nQS9JHWfQS1LHGfSS1HEGvSR1nEEvSR1n0EtSx/1/Tv2Hnu7uCTgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f7693d54cf8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "_ = plt.hist(final_params_flat, bins=200)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
